diff --git a/.devcontainer/postInstall.sh b/.devcontainer/postInstall.sh index cf3761a9..f2b12ea5 100755 --- a/.devcontainer/postInstall.sh +++ b/.devcontainer/postInstall.sh @@ -2,4 +2,4 @@ PATH=/home/vscode/.cargo/bin:$PATH cd dolma -source /home/vscode/miniforge3/bin/activate && pip install cmake "maturin[patchelf]>=1.1,<2.0" +source /home/vscode/miniforge3/bin/activate && pip install cmake "maturin>=1.5,<2.0" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 838abff0..1f007813 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -19,6 +19,7 @@ permissions: env: DOLMA_TESTS_SKIP_AWS: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'true' || 'false' }} DOLMA_TEST_S3_PREFIX: s3://dolma-tests + DOLMA_TEST_SKIP_LARGE_MODELS: "true" RUST_CHANNEL: stable jobs: diff --git a/Cargo.lock b/Cargo.lock index 1457f246..2beea7e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -987,7 +987,7 @@ dependencies = [ [[package]] name = "dolma" -version = "1.0.14" +version = "1.1.0" dependencies = [ "adblock", "ahash", diff --git a/Makefile b/Makefile index eab71919..7a508485 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,29 @@ +UNAME := $(shell uname) + +ifeq ($(UNAME), Darwin) + OS_MESSAGE := "MacOS detected" + CMAKE_SETUP := "which cmake || brew install cmake" + PROTOBUF_SETUP := "which protoc || brew install protobuf" + OPENSSL_SETUP := "which openssl || brew install openssl" +else ifeq ($(UNAME), Linux) + OS_MESSAGE := "Linux detected" + CMAKE_SETUP := "which cmake || sudo apt-get install --yes build-essential cmake" + PROTOBUF_SETUP := "which protoc || sudo apt-get install --yes protobuf-compiler" + OPENSSL_SETUP := "which openssl || sudo apt-get install --yes libssl-dev" +else + OS_MESSAGE := "Unsupported OS; please install rust, cmake, protobuf, and openssl manually" + CMAKE_SETUP := "" + PROTOBUF_SETUP := "" + OPENSSL_SETUP := "" +endif + setup: - @./setup.sh + @echo "${OS_MESSAGE}: installing..." + $(shell "${CMAKE_SETUP}") + $(shell "${PROTOBUF_SETUP}") + $(shell "${OPENSSL_SETUP}") + which cargo || curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + which maturin || pip install 'maturin>=1.5,<2.0' publish: maturin publish diff --git a/configs/cc-news/dedupe.sh b/configs/cc-news/dedupe.sh new file mode 100644 index 00000000..5e99f00d --- /dev/null +++ b/configs/cc-news/dedupe.sh @@ -0,0 +1,73 @@ +#! /usr/bin/env bash + +# documents: +# - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/*.gz + +# dedupe: +# name: dedupe_para_ngrams_13_1 +# paragraphs: +# attribute_name: dedupe_para_ngrams_13_1 +# by_ngram: +# ngram_length: 13 +# stride: 1 +# overlap_threshold: 0.5 +# skip_empty: true + +# bloom_filter: +# file: ${oc.env:HOME}/c4_dedupe_para_ngrams_13_1.bin +# read_only: false +# # estimated doc count is obtained by counting number of words in paragraphs +# # then dividing by 13 (ngram_length) and multiplying by 2 (for each ngram) +# estimated_doc_count: 359_916_731_334 +# desired_false_positive_rate: 0.1 + +# processes: 188 +# work_dir: +# input: /tmp/c4_dedupe_para_ngrams_13_1/input +# output: /tmp/c4_dedupe_para_ngrams_13_1/output + +# run years between 2016 and 2024 +for year in {2016..2024}; do + # run months between 1 and 12 + for month in {1..12}; do + # skip months after 7 if year is 2024 + if [ $year -eq 2024 ] && [ $month -gt 7 ]; then + continue + fi + + # skip months before 8 if year is 2016 + if [ $year -eq 2016 ] && [ $month -lt 8 ]; then + continue + fi + + # rename month to 2 digits + month=$(printf "%02d" $month) + + documents="s3://ai2-llm/pretraining-data/sources/cc-news/v0-resiliparse/documents/${year}-${month}/*.zst" + + size=$(aws s3api list-objects --bucket ai2-llm --prefix "pretraining-data/sources/cc-news/v0-resiliparse/documents/${year}-${month}/" --output json --query "[sum(Contents[].Size)]" | jq '.[0]' -rc) + + # run deduplication + echo "Running fuzzy dedupe for ${year}-${month} with ${size} bytes Bloom filter" + + set -ex + + dolma dedupe \ + --documents ${documents} \ + --dedupe.name dedupe_ngrams_13_1 \ + --dedupe.paragraphs.attribute_name dedupe_ngrams_13_1 \ + --dedupe.paragraphs.by_ngram.ngram_length 13 \ + --dedupe.paragraphs.by_ngram.stride 1 \ + --dedupe.paragraphs.by_ngram.overlap_threshold 0.5 \ + --dedupe.skip_empty \ + --bloom_filter.file "${HOME}/cc-news/dedupe_ngrams_13_1-${year}-${month}.bin" \ + --no-bloom_filter.read_only \ + --bloom_filter.estimated_doc_count $size \ + --bloom_filter.desired_false_positive_rate 0.01 \ + --processes "$(expr $(nproc) - 4)" \ + --work_dir.input /tmp/cc-news/dedupe_ngrams_13_1/${year}-${month}/input \ + --work_dir.output /tmp/cc-news/dedupe_ngrams_13_1/${year}-${month}/output + + set +ex + done +done diff --git a/configs/cc-news/extract.sh b/configs/cc-news/extract.sh new file mode 100644 index 00000000..c36c4272 --- /dev/null +++ b/configs/cc-news/extract.sh @@ -0,0 +1,43 @@ +#! /usr/bin/env bash + +# run years between 2016 and 2024 +for year in {2016..2024}; do + # run months between 1 and 12 + for month in {1..12}; do + # skip months after 7 if year is 2024 + if [ $year -eq 2024 ] && [ $month -gt 7 ]; then + continue + fi + + # skip months before 8 if year is 2016 + if [ $year -eq 2016 ] && [ $month -lt 8 ]; then + continue + fi + + # rename month to 2 digits + month=$(printf "%02d" $month) + + documents="s3://ai2-russella/crawl-data/CC-NEWS/${year}/${month}/*.warc.gz" + + # run the extraction + echo "Running extraction for ${year}-${month}" + + set -ex + + dolma warc \ + --documents ${documents} \ + --destination s3://ai2-llm/pretraining-data/sources/cc-news/v0-resiliparse/documents/${year}-${month} \ + --processes "$(expr $(nproc) - 4)" \ + --source_name cc-news_${year}-${month} \ + --linearizer resiliparse \ + --pre.taggers cc_re \ + --no-pre.skip \ + --no-store.html \ + --store.attr_spans 500 \ + --skip_duplicate_urls \ + --work_dir.input /tmp/cc-news/${year}-${month}/input \ + --work_dir.output /tmp/cc-news/${year}-${month}/output + + set +ex + done +done diff --git a/configs/cc-news/make_lang_partition.py b/configs/cc-news/make_lang_partition.py new file mode 100644 index 00000000..85b175e1 --- /dev/null +++ b/configs/cc-news/make_lang_partition.py @@ -0,0 +1,64 @@ +import json +from typing import List +import smart_open + + +SRC_BASE = "s3://ai2-llm/pretraining-data/sources/cc-news" +SRC_PRFX = "v1-resiliparse" +LANG_THR = 100_000 +DST_BASE = "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news" +DST_PRFX = f"v2-resiliparse-l{LANG_THR // 1000}k" + + +def base_stream_config(lang: str, year: int, months: List[int]): + return { + "name": f"cc-news_{year:04d}_{lang}", + "documents": [ + f"{SRC_BASE}/{SRC_PRFX}/documents/{year:04d}-{month:02d}/*.zst" + for month in months + ], + "compression": {"input": "zst", "output": "zst"}, + "output": { + "path": f"{DST_BASE}/{DST_PRFX}/documents/{lang}/{year:04d}", + "max_size_in_bytes": 10_000_000_000, + }, + "attributes": ["ft_lang_id_1e2", "dolma_v2_tokenizer"], + "filter": { + "include": [ + # at least 100 tokens + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + # make sure the language is present and the confidence is high enough and that it is the highest confidence + ( + f"(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__{lang} != null) and " + + f"(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__{lang}[0][-1] >= 0.5) and " + + f'((.attributes | to_entries | map(select(.key | startswith("ft_lang_id_1e2__ft_lang_id_1e2__"))) | max_by(.value) | .key ) == "ft_lang_id_1e2__ft_lang_id_1e2__{lang}")' + ), + ], + "syntax": "jq", + }, + } + + +def main(): + with smart_open.open("s3://ai2-llm/stats/cc-news/v1-resiliparse/attributes/ft_lang_id_1e2_summary.json") as f: + lang_counts = json.load(f) + + languages = {k: v for k, v in lang_counts.items() if v >= LANG_THR} + + streams = [] + for year in range(2016, 2025): + if year == 2016: + months = list(range(8, 13)) + elif year == 2024: + months = list(range(1, 8)) + else: + months = list(range(1, 13)) + + streams.extend([base_stream_config(lang, year, months) for lang in languages]) + + with smart_open.open("configs/cc-news/mix_v2.json", "wt") as f: + json.dump({"processes": 1, "streams": streams}, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/configs/cc-news/mix_v1.sh b/configs/cc-news/mix_v1.sh new file mode 100644 index 00000000..62cf779a --- /dev/null +++ b/configs/cc-news/mix_v1.sh @@ -0,0 +1,45 @@ +#! /usr/bin/env bash + + +# get script directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + SOURCE="$(readlink "$SOURCE")" + # if $SOURCE was a relative symlink, we need to resolve it + # relative to the path where the symlink file was located + [[ $SOURCE != /* ]] && SOURCE="$SCRIPT_DIR/$SOURCE" +done +SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + + +# run years between 2016 and 2024 +for year in {2016..2024}; do + # run months between 1 and 12 + for month in {1..12}; do + # skip months after 7 if year is 2024 + if [ $year -eq 2024 ] && [ $month -gt 7 ]; then + continue + fi + + # skip months before 8 if year is 2016 + if [ $year -eq 2016 ] && [ $month -lt 8 ]; then + continue + fi + + # rename month to 2 digits + month=$(printf "%02d" $month) + + # run deduplication + echo "Mixing ${year}-${month}" + + export MIX_MONTH=${month} + export MIX_YEAR=${year} + + set -ex + + dolma -c ${SCRIPT_DIR}/mix_v1.yaml mix + + set +ex + done +done diff --git a/configs/cc-news/mix_v1.yaml b/configs/cc-news/mix_v1.yaml new file mode 100644 index 00000000..4bb59a18 --- /dev/null +++ b/configs/cc-news/mix_v1.yaml @@ -0,0 +1,25 @@ +streams: + - name: cc-news_${oc.env:MIX_YEAR}-${oc.env:MIX_MONTH} + documents: + - s3://ai2-llm/pretraining-data/sources/cc-news/v0-resiliparse/documents/${oc.env:MIX_YEAR}-${oc.env:MIX_MONTH}/*.zst + + compression: + input: zst + output: zst + + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/${oc.env:MIX_YEAR}-${oc.env:MIX_MONTH} + max_size_in_bytes: 1_000_000_000 + + attributes: + - dedupe_ngrams_13_1 + + filter: + exclude: + - >- + (.attributes.dedupe_ngrams_13_1 | length > 0) and + ((.attributes.dedupe_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.5) + + syntax: jq + +processes: 1 diff --git a/configs/cc-news/mix_v2.json b/configs/cc-news/mix_v2.json new file mode 100644 index 00000000..b59009fe --- /dev/null +++ b/configs/cc-news/mix_v2.json @@ -0,0 +1,19349 @@ +{ + "processes": 1, + "streams": [ + { + "name": "cc-news_2016_en", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/en/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__en\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_es", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/es/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__es\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ru", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ru/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ru\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_it", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/it/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__it\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_de", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/de/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__de\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_fr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fr/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ar", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ar/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ar\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_hi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hi/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_pt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pt/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_tr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tr/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_zh", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/zh/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__zh\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_el", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/el/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__el\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ja", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ja/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ja\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ro", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ro/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ro\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_nl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nl/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ko", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ko/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ko\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_id", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/id/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__id\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_uk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/uk/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__uk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_vi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/vi/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__vi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_pl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pl/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_fa", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fa/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fa\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_sv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sv/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ta", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ta/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ta\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_bg", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bg/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bg\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_hu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hu/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_cs", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/cs/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__cs\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ur", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ur/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ur\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_no", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/no/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__no\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_fi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fi/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_bn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bn/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_et", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/et/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__et\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_mr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mr/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_hr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hr/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ml", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ml/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ml\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ca", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ca/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ca\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_te", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/te/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__te\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_da", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/da/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__da\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_sl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sl/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_sk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sk/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_sq", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sq/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sq\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_sr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sr/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_az", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/az/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__az\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_he", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/he/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__he\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_th", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/th/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__th\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_lt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lt/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_kn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kn/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_lv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lv/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_mk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mk/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_hy", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hy/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hy\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_is", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/is/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__is\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_kk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kk/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_gu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gu/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ka", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ka/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ka\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_nn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nn/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ne", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ne/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ne\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_gl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gl/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ckb", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ckb/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ckb\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ky", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ky/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ky\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_tl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tl/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_ug", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ug/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ug\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_be", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/be/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__be\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2016_eu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2016-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/eu/2016", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__eu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_en", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/en/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__en\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_es", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/es/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__es\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ru", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ru/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ru\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_it", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/it/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__it\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_de", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/de/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__de\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_fr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fr/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ar", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ar/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ar\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_hi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hi/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_pt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pt/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_tr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tr/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_zh", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/zh/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__zh\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_el", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/el/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__el\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ja", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ja/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ja\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ro", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ro/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ro\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_nl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nl/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ko", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ko/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ko\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_id", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/id/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__id\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_uk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/uk/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__uk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_vi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/vi/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__vi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_pl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pl/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_fa", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fa/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fa\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_sv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sv/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ta", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ta/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ta\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_bg", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bg/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bg\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_hu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hu/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_cs", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/cs/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__cs\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ur", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ur/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ur\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_no", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/no/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__no\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_fi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fi/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_bn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bn/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_et", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/et/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__et\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_mr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mr/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_hr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hr/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ml", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ml/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ml\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ca", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ca/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ca\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_te", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/te/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__te\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_da", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/da/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__da\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_sl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sl/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_sk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sk/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_sq", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sq/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sq\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_sr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sr/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_az", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/az/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__az\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_he", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/he/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__he\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_th", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/th/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__th\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_lt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lt/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_kn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kn/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_lv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lv/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_mk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mk/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_hy", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hy/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hy\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_is", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/is/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__is\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_kk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kk/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_gu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gu/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ka", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ka/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ka\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_nn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nn/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ne", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ne/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ne\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_gl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gl/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ckb", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ckb/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ckb\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ky", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ky/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ky\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_tl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tl/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_ug", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ug/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ug\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_be", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/be/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__be\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2017_eu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2017-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/eu/2017", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__eu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_en", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/en/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__en\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_es", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/es/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__es\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ru", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ru/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ru\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_it", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/it/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__it\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_de", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/de/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__de\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_fr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fr/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ar", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ar/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ar\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_hi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hi/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_pt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pt/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_tr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tr/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_zh", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/zh/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__zh\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_el", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/el/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__el\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ja", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ja/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ja\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ro", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ro/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ro\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_nl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nl/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ko", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ko/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ko\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_id", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/id/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__id\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_uk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/uk/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__uk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_vi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/vi/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__vi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_pl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pl/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_fa", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fa/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fa\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_sv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sv/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ta", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ta/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ta\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_bg", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bg/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bg\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_hu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hu/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_cs", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/cs/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__cs\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ur", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ur/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ur\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_no", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/no/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__no\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_fi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fi/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_bn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bn/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_et", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/et/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__et\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_mr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mr/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_hr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hr/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ml", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ml/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ml\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ca", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ca/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ca\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_te", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/te/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__te\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_da", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/da/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__da\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_sl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sl/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_sk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sk/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_sq", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sq/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sq\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_sr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sr/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_az", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/az/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__az\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_he", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/he/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__he\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_th", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/th/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__th\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_lt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lt/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_kn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kn/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_lv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lv/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_mk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mk/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_hy", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hy/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hy\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_is", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/is/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__is\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_kk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kk/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_gu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gu/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ka", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ka/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ka\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_nn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nn/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ne", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ne/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ne\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_gl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gl/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ckb", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ckb/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ckb\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ky", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ky/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ky\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_tl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tl/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_ug", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ug/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ug\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_be", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/be/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__be\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2018_eu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2018-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/eu/2018", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__eu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_en", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/en/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__en\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_es", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/es/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__es\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ru", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ru/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ru\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_it", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/it/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__it\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_de", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/de/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__de\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_fr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fr/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ar", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ar/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ar\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_hi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hi/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_pt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pt/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_tr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tr/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_zh", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/zh/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__zh\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_el", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/el/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__el\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ja", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ja/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ja\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ro", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ro/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ro\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_nl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nl/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ko", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ko/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ko\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_id", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/id/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__id\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_uk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/uk/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__uk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_vi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/vi/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__vi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_pl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pl/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_fa", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fa/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fa\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_sv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sv/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ta", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ta/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ta\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_bg", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bg/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bg\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_hu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hu/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_cs", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/cs/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__cs\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ur", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ur/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ur\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_no", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/no/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__no\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_fi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fi/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_bn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bn/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_et", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/et/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__et\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_mr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mr/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_hr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hr/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ml", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ml/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ml\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ca", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ca/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ca\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_te", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/te/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__te\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_da", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/da/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__da\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_sl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sl/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_sk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sk/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_sq", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sq/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sq\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_sr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sr/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_az", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/az/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__az\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_he", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/he/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__he\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_th", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/th/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__th\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_lt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lt/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_kn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kn/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_lv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lv/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_mk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mk/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_hy", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hy/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hy\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_is", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/is/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__is\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_kk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kk/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_gu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gu/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ka", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ka/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ka\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_nn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nn/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ne", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ne/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ne\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_gl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gl/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ckb", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ckb/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ckb\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ky", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ky/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ky\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_tl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tl/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_ug", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ug/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ug\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_be", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/be/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__be\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2019_eu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2019-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/eu/2019", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__eu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_en", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/en/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__en\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_es", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/es/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__es\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ru", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ru/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ru\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_it", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/it/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__it\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_de", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/de/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__de\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_fr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fr/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ar", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ar/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ar\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_hi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hi/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_pt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pt/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_tr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tr/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_zh", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/zh/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__zh\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_el", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/el/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__el\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ja", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ja/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ja\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ro", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ro/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ro\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_nl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nl/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ko", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ko/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ko\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_id", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/id/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__id\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_uk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/uk/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__uk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_vi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/vi/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__vi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_pl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pl/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_fa", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fa/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fa\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_sv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sv/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ta", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ta/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ta\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_bg", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bg/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bg\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_hu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hu/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_cs", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/cs/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__cs\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ur", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ur/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ur\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_no", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/no/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__no\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_fi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fi/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_bn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bn/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_et", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/et/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__et\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_mr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mr/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_hr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hr/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ml", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ml/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ml\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ca", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ca/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ca\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_te", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/te/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__te\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_da", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/da/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__da\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_sl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sl/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_sk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sk/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_sq", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sq/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sq\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_sr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sr/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_az", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/az/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__az\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_he", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/he/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__he\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_th", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/th/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__th\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_lt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lt/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_kn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kn/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_lv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lv/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_mk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mk/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_hy", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hy/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hy\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_is", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/is/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__is\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_kk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kk/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_gu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gu/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ka", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ka/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ka\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_nn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nn/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ne", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ne/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ne\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_gl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gl/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ckb", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ckb/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ckb\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ky", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ky/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ky\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_tl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tl/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_ug", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ug/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ug\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_be", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/be/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__be\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2020_eu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2020-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/eu/2020", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__eu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_en", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/en/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__en\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_es", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/es/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__es\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ru", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ru/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ru\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_it", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/it/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__it\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_de", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/de/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__de\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_fr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fr/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ar", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ar/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ar\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_hi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hi/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_pt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pt/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_tr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tr/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_zh", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/zh/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__zh\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_el", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/el/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__el\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ja", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ja/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ja\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ro", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ro/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ro\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_nl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nl/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ko", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ko/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ko\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_id", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/id/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__id\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_uk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/uk/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__uk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_vi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/vi/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__vi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_pl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pl/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_fa", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fa/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fa\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_sv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sv/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ta", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ta/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ta\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_bg", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bg/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bg\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_hu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hu/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_cs", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/cs/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__cs\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ur", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ur/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ur\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_no", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/no/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__no\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_fi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fi/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_bn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bn/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_et", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/et/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__et\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_mr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mr/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_hr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hr/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ml", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ml/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ml\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ca", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ca/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ca\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_te", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/te/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__te\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_da", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/da/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__da\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_sl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sl/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_sk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sk/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_sq", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sq/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sq\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_sr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sr/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_az", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/az/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__az\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_he", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/he/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__he\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_th", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/th/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__th\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_lt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lt/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_kn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kn/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_lv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lv/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_mk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mk/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_hy", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hy/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hy\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_is", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/is/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__is\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_kk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kk/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_gu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gu/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ka", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ka/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ka\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_nn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nn/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ne", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ne/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ne\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_gl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gl/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ckb", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ckb/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ckb\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ky", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ky/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ky\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_tl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tl/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_ug", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ug/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ug\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_be", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/be/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__be\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2021_eu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2021-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/eu/2021", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__eu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_en", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/en/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__en\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_es", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/es/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__es\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ru", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ru/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ru\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_it", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/it/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__it\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_de", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/de/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__de\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_fr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fr/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ar", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ar/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ar\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_hi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hi/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_pt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pt/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_tr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tr/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_zh", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/zh/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__zh\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_el", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/el/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__el\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ja", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ja/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ja\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ro", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ro/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ro\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_nl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nl/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ko", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ko/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ko\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_id", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/id/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__id\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_uk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/uk/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__uk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_vi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/vi/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__vi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_pl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pl/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_fa", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fa/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fa\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_sv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sv/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ta", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ta/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ta\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_bg", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bg/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bg\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_hu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hu/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_cs", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/cs/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__cs\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ur", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ur/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ur\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_no", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/no/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__no\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_fi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fi/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_bn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bn/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_et", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/et/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__et\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_mr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mr/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_hr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hr/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ml", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ml/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ml\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ca", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ca/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ca\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_te", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/te/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__te\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_da", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/da/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__da\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_sl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sl/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_sk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sk/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_sq", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sq/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sq\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_sr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sr/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_az", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/az/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__az\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_he", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/he/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__he\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_th", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/th/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__th\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_lt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lt/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_kn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kn/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_lv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lv/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_mk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mk/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_hy", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hy/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hy\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_is", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/is/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__is\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_kk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kk/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_gu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gu/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ka", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ka/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ka\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_nn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nn/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ne", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ne/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ne\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_gl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gl/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ckb", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ckb/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ckb\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ky", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ky/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ky\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_tl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tl/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_ug", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ug/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ug\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_be", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/be/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__be\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2022_eu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2022-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/eu/2022", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__eu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_en", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/en/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__en\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_es", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/es/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__es\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ru", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ru/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ru\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_it", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/it/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__it\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_de", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/de/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__de\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_fr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fr/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ar", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ar/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ar\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_hi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hi/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_pt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pt/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_tr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tr/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_zh", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/zh/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__zh\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_el", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/el/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__el\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ja", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ja/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ja\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ro", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ro/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ro\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_nl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nl/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ko", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ko/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ko\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_id", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/id/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__id\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_uk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/uk/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__uk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_vi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/vi/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__vi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_pl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pl/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_fa", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fa/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fa\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_sv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sv/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ta", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ta/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ta\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_bg", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bg/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bg\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_hu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hu/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_cs", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/cs/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__cs\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ur", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ur/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ur\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_no", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/no/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__no\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_fi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fi/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_bn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bn/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_et", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/et/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__et\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_mr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mr/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_hr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hr/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ml", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ml/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ml\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ca", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ca/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ca\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_te", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/te/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__te\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_da", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/da/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__da\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_sl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sl/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_sk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sk/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_sq", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sq/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sq\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_sr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sr/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_az", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/az/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__az\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_he", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/he/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__he\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_th", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/th/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__th\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_lt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lt/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_kn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kn/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_lv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lv/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_mk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mk/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_hy", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hy/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hy\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_is", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/is/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__is\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_kk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kk/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_gu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gu/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ka", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ka/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ka\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_nn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nn/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ne", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ne/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ne\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_gl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gl/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ckb", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ckb/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ckb\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ky", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ky/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ky\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_tl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tl/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_ug", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ug/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ug\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_be", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/be/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__be\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2023_eu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-07/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-08/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-09/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-10/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-11/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2023-12/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/eu/2023", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__eu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_en", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/en/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__en\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_es", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/es/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__es[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__es\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ru", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ru/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ru[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ru\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_it", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/it/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__it[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__it\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_de", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/de/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__de[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__de\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_fr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fr/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ar", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ar/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ar[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ar\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_hi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hi/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_pt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pt/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_tr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tr/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_zh", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/zh/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__zh[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__zh\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_el", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/el/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__el[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__el\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ja", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ja/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ja[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ja\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ro", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ro/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ro[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ro\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_nl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nl/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ko", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ko/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ko[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ko\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_id", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/id/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__id[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__id\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_uk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/uk/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__uk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__uk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_vi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/vi/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__vi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__vi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_pl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/pl/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__pl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__pl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_fa", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fa/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fa[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fa\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_sv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sv/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ta", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ta/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ta[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ta\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_bg", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bg/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bg[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bg\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_hu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hu/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_cs", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/cs/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__cs[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__cs\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ur", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ur/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ur[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ur\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_no", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/no/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__no[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__no\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_fi", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/fi/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__fi[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__fi\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_bn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/bn/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__bn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__bn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_et", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/et/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__et[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__et\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_mr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mr/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_hr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hr/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ml", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ml/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ml[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ml\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ca", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ca/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ca[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ca\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_te", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/te/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__te[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__te\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_da", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/da/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__da[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__da\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_sl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sl/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_sk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sk/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_sq", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sq/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sq[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sq\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_sr", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/sr/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__sr[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__sr\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_az", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/az/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__az[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__az\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_he", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/he/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__he[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__he\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_th", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/th/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__th[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__th\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_lt", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lt/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lt[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lt\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_kn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kn/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_lv", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/lv/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__lv[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__lv\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_mk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/mk/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__mk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__mk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_hy", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/hy/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__hy[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__hy\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_is", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/is/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__is[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__is\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_kk", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/kk/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__kk[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__kk\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_gu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gu/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gu\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ka", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ka/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ka[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ka\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_nn", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/nn/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__nn[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__nn\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ne", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ne/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ne[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ne\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_gl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/gl/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__gl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__gl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ckb", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ckb/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ckb[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ckb\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ky", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ky/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ky[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ky\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_tl", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/tl/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__tl[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__tl\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_ug", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/ug/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__ug[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__ug\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_be", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/be/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__be[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__be\")" + ], + "syntax": "jq" + } + }, + { + "name": "cc-news_2024_eu", + "documents": [ + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-01/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-02/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-03/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-04/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-05/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-06/*.zst", + "s3://ai2-llm/pretraining-data/sources/cc-news/v1-resiliparse/documents/2024-07/*.zst" + ], + "compression": { + "input": "zst", + "output": "zst" + }, + "output": { + "path": "${oc.env:HOME}/ai2-llm/pretraining-data/sources/cc-news/v2-resiliparse-l100k/documents/eu/2024", + "max_size_in_bytes": 10000000000 + }, + "attributes": [ + "ft_lang_id_1e2", + "dolma_v2_tokenizer" + ], + "filter": { + "include": [ + ".attributes.dolma_v2_tokenizer__dolma_v2_tokenizer__length[0][-1] >= 100", + "(.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu != null) and (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__eu[0][-1] >= 0.5) and ((.attributes | to_entries | map(select(.key | startswith(\"ft_lang_id_1e2__ft_lang_id_1e2__\"))) | max_by(.value) | .key ) == \"ft_lang_id_1e2__ft_lang_id_1e2__eu\")" + ], + "syntax": "jq" + } + } + ] +} \ No newline at end of file diff --git a/configs/cccc/2024-2023/taggers.yaml b/configs/cccc/2024-2023/taggers.yaml new file mode 100644 index 00000000..a07e3dcd --- /dev/null +++ b/configs/cccc/2024-2023/taggers.yaml @@ -0,0 +1,7 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/*/warc/*zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/*/warc/*zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/*/warc/*zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-40/*/warc/*zst + +taggers: diff --git a/configs/cccc/bloom_test.yaml b/configs/cccc/bloom_test.yaml new file mode 100644 index 00000000..9facc655 --- /dev/null +++ b/configs/cccc/bloom_test.yaml @@ -0,0 +1,25 @@ + +documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/cccc/v2/documents/CC-MAIN-2022-05/*.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/*.gz + +dedupe: + name: dedupe_para_test + paragraphs: + attribute_name: dedupe_para_test + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/bloom_test/filter.bloom + # read_only: false + read_only: true + estimated_doc_count: 10_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2013-20.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2013-20.yaml new file mode 100644 index 00000000..1f2be0d5 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2013-20.yaml @@ -0,0 +1,49 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0027.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2013-20/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2013-20/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2013-20/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2013-48.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2013-48.yaml new file mode 100644 index 00000000..0ddce48b --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2013-48.yaml @@ -0,0 +1,54 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0032.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2013-48/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2013-48/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2013-48/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2014-10.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2014-10.yaml new file mode 100644 index 00000000..97423809 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2014-10.yaml @@ -0,0 +1,58 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0036.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2014-10/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-10/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-10/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2014-15.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2014-15.yaml new file mode 100644 index 00000000..01c928ee --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2014-15.yaml @@ -0,0 +1,67 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0045.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2014-15/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-15/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-15/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2014-23.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2014-23.yaml new file mode 100644 index 00000000..bcd5add5 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2014-23.yaml @@ -0,0 +1,75 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0052.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0053.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2014-23/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-23/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-23/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2014-35.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2014-35.yaml new file mode 100644 index 00000000..a83ea591 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2014-35.yaml @@ -0,0 +1,65 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0043.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2014-35/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-35/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-35/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2014-41.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2014-41.yaml new file mode 100644 index 00000000..e5c1813f --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2014-41.yaml @@ -0,0 +1,63 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0041.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2014-41/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-41/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-41/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2014-42.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2014-42.yaml new file mode 100644 index 00000000..4c5ddfd7 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2014-42.yaml @@ -0,0 +1,86 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0052.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0053.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0054.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0055.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0056.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0057.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0058.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0059.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0060.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0061.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0062.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0063.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0064.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2014-42/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-42/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-42/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2014-49.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2014-49.yaml new file mode 100644 index 00000000..b47ada8f --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2014-49.yaml @@ -0,0 +1,54 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0032.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2014-49/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-49/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-49/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2014-52.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2014-52.yaml new file mode 100644 index 00000000..eca48184 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2014-52.yaml @@ -0,0 +1,55 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0033.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2014-52/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-52/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2014-52/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2015-06.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2015-06.yaml new file mode 100644 index 00000000..c92a33c2 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2015-06.yaml @@ -0,0 +1,49 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0027.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2015-06/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-06/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-06/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2015-11.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2015-11.yaml new file mode 100644 index 00000000..91d36b24 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2015-11.yaml @@ -0,0 +1,54 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0032.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2015-11/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-11/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-11/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2015-14.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2015-14.yaml new file mode 100644 index 00000000..27fc4bd4 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2015-14.yaml @@ -0,0 +1,51 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0029.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2015-14/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-14/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-14/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2015-18.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2015-18.yaml new file mode 100644 index 00000000..1d3b0e06 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2015-18.yaml @@ -0,0 +1,59 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0037.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2015-18/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-18/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-18/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2015-22.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2015-22.yaml new file mode 100644 index 00000000..2a08733f --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2015-22.yaml @@ -0,0 +1,58 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0036.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2015-22/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-22/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-22/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2015-27.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2015-27.yaml new file mode 100644 index 00000000..5f5c3fba --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2015-27.yaml @@ -0,0 +1,52 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0030.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2015-27/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-27/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2015-27/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2016-07.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2016-07.yaml new file mode 100644 index 00000000..ea88645f --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2016-07.yaml @@ -0,0 +1,56 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0034.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2016-07/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2016-07/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2016-07/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2016-18.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2016-18.yaml new file mode 100644 index 00000000..bb80a56f --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2016-18.yaml @@ -0,0 +1,47 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0025.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2016-18/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2016-18/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2016-18/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2016-22.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2016-22.yaml new file mode 100644 index 00000000..04ef7663 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2016-22.yaml @@ -0,0 +1,46 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0024.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2016-22/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2016-22/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2016-22/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2016-26.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2016-26.yaml new file mode 100644 index 00000000..2f89dc3b --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2016-26.yaml @@ -0,0 +1,22 @@ + +documents: + + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2016-26/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2016-26/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2016-26/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2016-30.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2016-30.yaml new file mode 100644 index 00000000..1ca14595 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2016-30.yaml @@ -0,0 +1,22 @@ + +documents: + + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2016-30/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2016-30/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2016-30/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2017-04.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2017-04.yaml new file mode 100644 index 00000000..658a73f0 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2017-04.yaml @@ -0,0 +1,58 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0036.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2017-04/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-04/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-04/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2017-09.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2017-09.yaml new file mode 100644 index 00000000..75f719e0 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2017-09.yaml @@ -0,0 +1,65 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0043.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2017-09/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-09/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-09/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2017-13.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2017-13.yaml new file mode 100644 index 00000000..d73fadab --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2017-13.yaml @@ -0,0 +1,70 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0048.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2017-13/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-13/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-13/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2017-17.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2017-17.yaml new file mode 100644 index 00000000..7264c4a0 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2017-17.yaml @@ -0,0 +1,72 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0050.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2017-17/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-17/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-17/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2017-22.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2017-22.yaml new file mode 100644 index 00000000..3335da2a --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2017-22.yaml @@ -0,0 +1,70 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0048.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2017-22/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-22/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-22/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2017-26.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2017-26.yaml new file mode 100644 index 00000000..98882292 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2017-26.yaml @@ -0,0 +1,75 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0052.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0053.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2017-26/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-26/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-26/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2017-30.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2017-30.yaml new file mode 100644 index 00000000..b48118fc --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2017-30.yaml @@ -0,0 +1,22 @@ + +documents: + + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2017-30/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-30/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-30/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2017-51.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2017-51.yaml new file mode 100644 index 00000000..a4f67d5f --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2017-51.yaml @@ -0,0 +1,42 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0020.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2017-51/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-51/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2017-51/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2018-09.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2018-09.yaml new file mode 100644 index 00000000..3b39b605 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2018-09.yaml @@ -0,0 +1,22 @@ + +documents: + + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2018-09/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-09/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-09/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2018-13.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2018-13.yaml new file mode 100644 index 00000000..40d515d8 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2018-13.yaml @@ -0,0 +1,64 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0042.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2018-13/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-13/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-13/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2018-22.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2018-22.yaml new file mode 100644 index 00000000..4a1df0fc --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2018-22.yaml @@ -0,0 +1,57 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0035.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2018-22/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-22/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-22/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2018-26.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2018-26.yaml new file mode 100644 index 00000000..e210acf4 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2018-26.yaml @@ -0,0 +1,62 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0040.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2018-26/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-26/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-26/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2018-30.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2018-30.yaml new file mode 100644 index 00000000..c7fb383a --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2018-30.yaml @@ -0,0 +1,66 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0044.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2018-30/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-30/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-30/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2018-34.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2018-34.yaml new file mode 100644 index 00000000..bca8a9fb --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2018-34.yaml @@ -0,0 +1,52 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0030.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2018-34/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-34/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-34/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2018-47.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2018-47.yaml new file mode 100644 index 00000000..4a599f4a --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2018-47.yaml @@ -0,0 +1,52 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0030.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2018-47/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-47/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-47/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2018-51.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2018-51.yaml new file mode 100644 index 00000000..f1729843 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2018-51.yaml @@ -0,0 +1,57 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0035.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2018-51/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-51/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2018-51/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2019-04.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2019-04.yaml new file mode 100644 index 00000000..9e246ee5 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2019-04.yaml @@ -0,0 +1,57 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0035.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2019-04/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-04/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-04/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2019-09.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2019-09.yaml new file mode 100644 index 00000000..b653f5e9 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2019-09.yaml @@ -0,0 +1,56 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0034.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2019-09/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-09/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-09/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2019-13.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2019-13.yaml new file mode 100644 index 00000000..24cfdf6a --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2019-13.yaml @@ -0,0 +1,52 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0030.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2019-13/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-13/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-13/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2019-30.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2019-30.yaml new file mode 100644 index 00000000..df7b2061 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2019-30.yaml @@ -0,0 +1,62 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0040.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2019-30/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-30/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-30/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2019-35.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2019-35.yaml new file mode 100644 index 00000000..68028359 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2019-35.yaml @@ -0,0 +1,72 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0050.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2019-35/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-35/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-35/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2019-39.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2019-39.yaml new file mode 100644 index 00000000..dc3ae657 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2019-39.yaml @@ -0,0 +1,68 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0046.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2019-39/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-39/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-39/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2019-43.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2019-43.yaml new file mode 100644 index 00000000..b80c95ef --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2019-43.yaml @@ -0,0 +1,22 @@ + +documents: + + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2019-43/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-43/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-43/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2019-51.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2019-51.yaml new file mode 100644 index 00000000..b710f29e --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2019-51.yaml @@ -0,0 +1,22 @@ + +documents: + + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2019-51/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-51/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2019-51/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2020-10.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2020-10.yaml new file mode 100644 index 00000000..324133a4 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2020-10.yaml @@ -0,0 +1,22 @@ + +documents: + + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2020-10/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2020-10/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2020-10/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2020-24.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2020-24.yaml new file mode 100644 index 00000000..7f629cc8 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2020-24.yaml @@ -0,0 +1,22 @@ + +documents: + + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2020-24/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2020-24/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2020-24/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2020-29.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2020-29.yaml new file mode 100644 index 00000000..3896ff67 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2020-29.yaml @@ -0,0 +1,74 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0052.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2020-29/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2020-29/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2020-29/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2020-34.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2020-34.yaml new file mode 100644 index 00000000..767f781e --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2020-34.yaml @@ -0,0 +1,68 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0046.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2020-34/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2020-34/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2020-34/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2020-40.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2020-40.yaml new file mode 100644 index 00000000..edb7498e --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2020-40.yaml @@ -0,0 +1,22 @@ + +documents: + + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2020-40/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2020-40/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2020-40/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2021-17.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2021-17.yaml new file mode 100644 index 00000000..06540a6f --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2021-17.yaml @@ -0,0 +1,38 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0016.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2021-17/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2021-17/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2021-17/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2021-39.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2021-39.yaml new file mode 100644 index 00000000..cf58d281 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2021-39.yaml @@ -0,0 +1,63 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0041.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2021-39/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2021-39/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2021-39/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2021-43.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2021-43.yaml new file mode 100644 index 00000000..8404732a --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2021-43.yaml @@ -0,0 +1,70 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0048.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2021-43/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2021-43/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2021-43/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2021-49.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2021-49.yaml new file mode 100644 index 00000000..5626147f --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2021-49.yaml @@ -0,0 +1,58 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0036.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2021-49/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2021-49/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2021-49/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2022-05.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2022-05.yaml new file mode 100644 index 00000000..cc448859 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2022-05.yaml @@ -0,0 +1,63 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0041.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2022-05/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2022-05/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2022-05/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2023-06.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2023-06.yaml new file mode 100644 index 00000000..30ea0b9d --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2023-06.yaml @@ -0,0 +1,67 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0045.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2023-06/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2023-06/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2023-06/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2023-14.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2023-14.yaml new file mode 100644 index 00000000..de277b65 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2023-14.yaml @@ -0,0 +1,64 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0042.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2023-14/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2023-14/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2023-14/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2023-23.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2023-23.yaml new file mode 100644 index 00000000..22302106 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2023-23.yaml @@ -0,0 +1,64 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0042.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2023-23/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2023-23/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2023-23/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2023-40.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2023-40.yaml new file mode 100644 index 00000000..b819a7c4 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2023-40.yaml @@ -0,0 +1,22 @@ + +documents: + + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2023-40/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2023-40/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2023-40/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2023-50.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2023-50.yaml new file mode 100644 index 00000000..9f0ade93 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2023-50.yaml @@ -0,0 +1,65 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0043.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2023-50/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2023-50/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2023-50/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2024-10.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2024-10.yaml new file mode 100644 index 00000000..80ed25b0 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2024-10.yaml @@ -0,0 +1,60 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0038.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2024-10/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2024-10/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2024-10/output" diff --git a/configs/cccc/dedupe-doc/CC-MAIN-2024-18.yaml b/configs/cccc/dedupe-doc/CC-MAIN-2024-18.yaml new file mode 100644 index 00000000..de6dd2e4 --- /dev/null +++ b/configs/cccc/dedupe-doc/CC-MAIN-2024-18.yaml @@ -0,0 +1,57 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0035.json.gz + +dedupe: + name: dedupe_doc + documents: + attribute_name: dedupe_doc + key: $.text + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-doc/cccc-CC-MAIN-2024-18/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-05 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2024-18/input" + output: "${oc.env:HOME}/status/dedupe-doc-v1/cccc-CC-MAIN-2024-18/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2013-20.yaml b/configs/cccc/dedupe-para/CC-MAIN-2013-20.yaml new file mode 100644 index 00000000..4331ff5a --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2013-20.yaml @@ -0,0 +1,53 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0027.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2013-20/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2013-20/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2013-20/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2013-48.yaml b/configs/cccc/dedupe-para/CC-MAIN-2013-48.yaml new file mode 100644 index 00000000..a09fe02b --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2013-48.yaml @@ -0,0 +1,58 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0032.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2013-48/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2013-48/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2013-48/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2014-10.yaml b/configs/cccc/dedupe-para/CC-MAIN-2014-10.yaml new file mode 100644 index 00000000..84a65226 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2014-10.yaml @@ -0,0 +1,62 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0036.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2014-10/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-10/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-10/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2014-15.yaml b/configs/cccc/dedupe-para/CC-MAIN-2014-15.yaml new file mode 100644 index 00000000..7d990392 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2014-15.yaml @@ -0,0 +1,71 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0045.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2014-15/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-15/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-15/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2014-23.yaml b/configs/cccc/dedupe-para/CC-MAIN-2014-23.yaml new file mode 100644 index 00000000..4add12ee --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2014-23.yaml @@ -0,0 +1,79 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0052.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0053.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2014-23/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-23/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-23/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2014-35.yaml b/configs/cccc/dedupe-para/CC-MAIN-2014-35.yaml new file mode 100644 index 00000000..a800ef50 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2014-35.yaml @@ -0,0 +1,69 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0043.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2014-35/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-35/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-35/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2014-41.yaml b/configs/cccc/dedupe-para/CC-MAIN-2014-41.yaml new file mode 100644 index 00000000..fed060b6 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2014-41.yaml @@ -0,0 +1,67 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0041.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2014-41/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-41/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-41/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2014-42.yaml b/configs/cccc/dedupe-para/CC-MAIN-2014-42.yaml new file mode 100644 index 00000000..73b679a3 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2014-42.yaml @@ -0,0 +1,90 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0052.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0053.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0054.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0055.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0056.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0057.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0058.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0059.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0060.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0061.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0062.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0063.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0064.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2014-42/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-42/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-42/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2014-49.yaml b/configs/cccc/dedupe-para/CC-MAIN-2014-49.yaml new file mode 100644 index 00000000..30fe1348 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2014-49.yaml @@ -0,0 +1,58 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0032.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2014-49/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-49/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-49/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2014-52.yaml b/configs/cccc/dedupe-para/CC-MAIN-2014-52.yaml new file mode 100644 index 00000000..38e2fcce --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2014-52.yaml @@ -0,0 +1,59 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0033.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2014-52/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-52/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2014-52/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2015-06.yaml b/configs/cccc/dedupe-para/CC-MAIN-2015-06.yaml new file mode 100644 index 00000000..7d5e93ec --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2015-06.yaml @@ -0,0 +1,53 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0027.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2015-06/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-06/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-06/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2015-11.yaml b/configs/cccc/dedupe-para/CC-MAIN-2015-11.yaml new file mode 100644 index 00000000..49948312 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2015-11.yaml @@ -0,0 +1,58 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0032.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2015-11/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-11/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-11/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2015-14.yaml b/configs/cccc/dedupe-para/CC-MAIN-2015-14.yaml new file mode 100644 index 00000000..78a956f3 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2015-14.yaml @@ -0,0 +1,55 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0029.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2015-14/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-14/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-14/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2015-18.yaml b/configs/cccc/dedupe-para/CC-MAIN-2015-18.yaml new file mode 100644 index 00000000..ff647514 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2015-18.yaml @@ -0,0 +1,63 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0037.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2015-18/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-18/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-18/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2015-22.yaml b/configs/cccc/dedupe-para/CC-MAIN-2015-22.yaml new file mode 100644 index 00000000..f0440bfb --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2015-22.yaml @@ -0,0 +1,62 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0036.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2015-22/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-22/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-22/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2015-27.yaml b/configs/cccc/dedupe-para/CC-MAIN-2015-27.yaml new file mode 100644 index 00000000..ce676f83 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2015-27.yaml @@ -0,0 +1,56 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0030.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2015-27/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-27/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2015-27/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2016-07.yaml b/configs/cccc/dedupe-para/CC-MAIN-2016-07.yaml new file mode 100644 index 00000000..2524b595 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2016-07.yaml @@ -0,0 +1,60 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0034.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2016-07/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2016-07/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2016-07/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2016-18.yaml b/configs/cccc/dedupe-para/CC-MAIN-2016-18.yaml new file mode 100644 index 00000000..ee3d203a --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2016-18.yaml @@ -0,0 +1,51 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0025.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2016-18/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2016-18/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2016-18/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2016-22.yaml b/configs/cccc/dedupe-para/CC-MAIN-2016-22.yaml new file mode 100644 index 00000000..901921e2 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2016-22.yaml @@ -0,0 +1,50 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0024.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2016-22/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2016-22/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2016-22/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2016-26.yaml b/configs/cccc/dedupe-para/CC-MAIN-2016-26.yaml new file mode 100644 index 00000000..2611b7f8 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2016-26.yaml @@ -0,0 +1,26 @@ + +documents: + + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2016-26/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2016-26/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2016-26/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2016-30.yaml b/configs/cccc/dedupe-para/CC-MAIN-2016-30.yaml new file mode 100644 index 00000000..76c611c5 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2016-30.yaml @@ -0,0 +1,26 @@ + +documents: + + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2016-30/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2016-30/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2016-30/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2017-04.yaml b/configs/cccc/dedupe-para/CC-MAIN-2017-04.yaml new file mode 100644 index 00000000..4c551206 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2017-04.yaml @@ -0,0 +1,62 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0036.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2017-04/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-04/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-04/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2017-09.yaml b/configs/cccc/dedupe-para/CC-MAIN-2017-09.yaml new file mode 100644 index 00000000..a5a69af3 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2017-09.yaml @@ -0,0 +1,69 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0043.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2017-09/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-09/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-09/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2017-13.yaml b/configs/cccc/dedupe-para/CC-MAIN-2017-13.yaml new file mode 100644 index 00000000..6dd679d0 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2017-13.yaml @@ -0,0 +1,74 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0048.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2017-13/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-13/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-13/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2017-17.yaml b/configs/cccc/dedupe-para/CC-MAIN-2017-17.yaml new file mode 100644 index 00000000..f1fd0c3a --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2017-17.yaml @@ -0,0 +1,76 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0050.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2017-17/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-17/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-17/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2017-22.yaml b/configs/cccc/dedupe-para/CC-MAIN-2017-22.yaml new file mode 100644 index 00000000..3cba6e05 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2017-22.yaml @@ -0,0 +1,74 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0048.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2017-22/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-22/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-22/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2017-26.yaml b/configs/cccc/dedupe-para/CC-MAIN-2017-26.yaml new file mode 100644 index 00000000..08822de9 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2017-26.yaml @@ -0,0 +1,79 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0052.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0053.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2017-26/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-26/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-26/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2017-30.yaml b/configs/cccc/dedupe-para/CC-MAIN-2017-30.yaml new file mode 100644 index 00000000..990c44c9 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2017-30.yaml @@ -0,0 +1,26 @@ + +documents: + + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2017-30/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-30/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-30/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2017-51.yaml b/configs/cccc/dedupe-para/CC-MAIN-2017-51.yaml new file mode 100644 index 00000000..feb04e17 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2017-51.yaml @@ -0,0 +1,46 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0020.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2017-51/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-51/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2017-51/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2018-09.yaml b/configs/cccc/dedupe-para/CC-MAIN-2018-09.yaml new file mode 100644 index 00000000..184fdd06 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2018-09.yaml @@ -0,0 +1,26 @@ + +documents: + + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2018-09/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-09/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-09/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2018-13.yaml b/configs/cccc/dedupe-para/CC-MAIN-2018-13.yaml new file mode 100644 index 00000000..44142f19 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2018-13.yaml @@ -0,0 +1,68 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0042.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2018-13/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-13/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-13/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2018-22.yaml b/configs/cccc/dedupe-para/CC-MAIN-2018-22.yaml new file mode 100644 index 00000000..27fce364 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2018-22.yaml @@ -0,0 +1,61 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0035.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2018-22/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-22/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-22/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2018-26.yaml b/configs/cccc/dedupe-para/CC-MAIN-2018-26.yaml new file mode 100644 index 00000000..687c1d86 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2018-26.yaml @@ -0,0 +1,66 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0040.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2018-26/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-26/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-26/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2018-26CC-MAIN-2021-43CC-MAIN-2017-51CC-MAIN-2019-39CC-MAIN-2023-50CC-MAIN-2018-13CC-MAIN-2016-18CC-MAIN-2016-07CC-MAIN-2023-23CC-MAIN-2018-22CC-MAIN-2019-04CC-MAIN-2017-22CC-MAIN-2023-06.yaml b/configs/cccc/dedupe-para/CC-MAIN-2018-26CC-MAIN-2021-43CC-MAIN-2017-51CC-MAIN-2019-39CC-MAIN-2023-50CC-MAIN-2018-13CC-MAIN-2016-18CC-MAIN-2016-07CC-MAIN-2023-23CC-MAIN-2018-22CC-MAIN-2019-04CC-MAIN-2017-22CC-MAIN-2023-06.yaml new file mode 100644 index 00000000..bae34090 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2018-26CC-MAIN-2021-43CC-MAIN-2017-51CC-MAIN-2019-39CC-MAIN-2023-50CC-MAIN-2018-13CC-MAIN-2016-18CC-MAIN-2016-07CC-MAIN-2023-23CC-MAIN-2018-22CC-MAIN-2019-04CC-MAIN-2017-22CC-MAIN-2023-06.yaml @@ -0,0 +1,26 @@ + +documents: + + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2018-26CC-MAIN-2021-43CC-MAIN-2017-51CC-MAIN-2019-39CC-MAIN-2023-50CC-MAIN-2018-13CC-MAIN-2016-18CC-MAIN-2016-07CC-MAIN-2023-23CC-MAIN-2018-22CC-MAIN-2019-04CC-MAIN-2017-22CC-MAIN-2023-06/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: /tmp/dedupe-para-v1/cccc-CC-MAIN-2018-26CC-MAIN-2021-43CC-MAIN-2017-51CC-MAIN-2019-39CC-MAIN-2023-50CC-MAIN-2018-13CC-MAIN-2016-18CC-MAIN-2016-07CC-MAIN-2023-23CC-MAIN-2018-22CC-MAIN-2019-04CC-MAIN-2017-22CC-MAIN-2023-06/input + output: /tmp/dedupe-para-v1/cccc-CC-MAIN-2018-26CC-MAIN-2021-43CC-MAIN-2017-51CC-MAIN-2019-39CC-MAIN-2023-50CC-MAIN-2018-13CC-MAIN-2016-18CC-MAIN-2016-07CC-MAIN-2023-23CC-MAIN-2018-22CC-MAIN-2019-04CC-MAIN-2017-22CC-MAIN-2023-06/output diff --git a/configs/cccc/dedupe-para/CC-MAIN-2018-30.yaml b/configs/cccc/dedupe-para/CC-MAIN-2018-30.yaml new file mode 100644 index 00000000..a96e757e --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2018-30.yaml @@ -0,0 +1,70 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0044.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2018-30/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-30/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-30/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2018-34.yaml b/configs/cccc/dedupe-para/CC-MAIN-2018-34.yaml new file mode 100644 index 00000000..c51d7094 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2018-34.yaml @@ -0,0 +1,56 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0030.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2018-34/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-34/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-34/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2018-47.yaml b/configs/cccc/dedupe-para/CC-MAIN-2018-47.yaml new file mode 100644 index 00000000..1e2cda9b --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2018-47.yaml @@ -0,0 +1,56 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0030.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2018-47/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-47/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-47/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2018-51.yaml b/configs/cccc/dedupe-para/CC-MAIN-2018-51.yaml new file mode 100644 index 00000000..224d53c6 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2018-51.yaml @@ -0,0 +1,61 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0035.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2018-51/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-51/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2018-51/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2019-04.yaml b/configs/cccc/dedupe-para/CC-MAIN-2019-04.yaml new file mode 100644 index 00000000..4a6b3a18 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2019-04.yaml @@ -0,0 +1,61 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0035.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2019-04/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-04/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-04/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2019-09.yaml b/configs/cccc/dedupe-para/CC-MAIN-2019-09.yaml new file mode 100644 index 00000000..76dc103e --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2019-09.yaml @@ -0,0 +1,60 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0034.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2019-09/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-09/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-09/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2019-13.yaml b/configs/cccc/dedupe-para/CC-MAIN-2019-13.yaml new file mode 100644 index 00000000..cafd3a91 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2019-13.yaml @@ -0,0 +1,56 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0030.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2019-13/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-13/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-13/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2019-30.yaml b/configs/cccc/dedupe-para/CC-MAIN-2019-30.yaml new file mode 100644 index 00000000..6ce6307a --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2019-30.yaml @@ -0,0 +1,66 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0040.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2019-30/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-30/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-30/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2019-35.yaml b/configs/cccc/dedupe-para/CC-MAIN-2019-35.yaml new file mode 100644 index 00000000..b270e274 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2019-35.yaml @@ -0,0 +1,76 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0050.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2019-35/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-35/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-35/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2019-39.yaml b/configs/cccc/dedupe-para/CC-MAIN-2019-39.yaml new file mode 100644 index 00000000..e551219e --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2019-39.yaml @@ -0,0 +1,72 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0046.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2019-39/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-39/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-39/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2019-43.yaml b/configs/cccc/dedupe-para/CC-MAIN-2019-43.yaml new file mode 100644 index 00000000..329893a5 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2019-43.yaml @@ -0,0 +1,26 @@ + +documents: + + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2019-43/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-43/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-43/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2019-51.yaml b/configs/cccc/dedupe-para/CC-MAIN-2019-51.yaml new file mode 100644 index 00000000..3f5fb0a9 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2019-51.yaml @@ -0,0 +1,26 @@ + +documents: + + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2019-51/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-51/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2019-51/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2020-10.yaml b/configs/cccc/dedupe-para/CC-MAIN-2020-10.yaml new file mode 100644 index 00000000..643f3d4d --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2020-10.yaml @@ -0,0 +1,26 @@ + +documents: + + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2020-10/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2020-10/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2020-10/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2020-24.yaml b/configs/cccc/dedupe-para/CC-MAIN-2020-24.yaml new file mode 100644 index 00000000..fc9f8a51 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2020-24.yaml @@ -0,0 +1,26 @@ + +documents: + + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2020-24/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2020-24/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2020-24/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2020-29.yaml b/configs/cccc/dedupe-para/CC-MAIN-2020-29.yaml new file mode 100644 index 00000000..3ee43ddd --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2020-29.yaml @@ -0,0 +1,78 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0052.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2020-29/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2020-29/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2020-29/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2020-34.yaml b/configs/cccc/dedupe-para/CC-MAIN-2020-34.yaml new file mode 100644 index 00000000..dbdaa4dc --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2020-34.yaml @@ -0,0 +1,72 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0046.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2020-34/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2020-34/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2020-34/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2020-40.yaml b/configs/cccc/dedupe-para/CC-MAIN-2020-40.yaml new file mode 100644 index 00000000..b7597262 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2020-40.yaml @@ -0,0 +1,26 @@ + +documents: + + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2020-40/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2020-40/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2020-40/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2021-17.yaml b/configs/cccc/dedupe-para/CC-MAIN-2021-17.yaml new file mode 100644 index 00000000..9a6656c3 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2021-17.yaml @@ -0,0 +1,42 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0016.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2021-17/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2021-17/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2021-17/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2021-39.yaml b/configs/cccc/dedupe-para/CC-MAIN-2021-39.yaml new file mode 100644 index 00000000..7ee5edc7 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2021-39.yaml @@ -0,0 +1,67 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0041.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2021-39/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2021-39/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2021-39/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2021-43.yaml b/configs/cccc/dedupe-para/CC-MAIN-2021-43.yaml new file mode 100644 index 00000000..f94d8dea --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2021-43.yaml @@ -0,0 +1,74 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0048.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2021-43/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2021-43/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2021-43/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2021-49.yaml b/configs/cccc/dedupe-para/CC-MAIN-2021-49.yaml new file mode 100644 index 00000000..f473a2c2 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2021-49.yaml @@ -0,0 +1,62 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0036.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2021-49/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2021-49/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2021-49/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2022-05.yaml b/configs/cccc/dedupe-para/CC-MAIN-2022-05.yaml new file mode 100644 index 00000000..0b75b1a2 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2022-05.yaml @@ -0,0 +1,67 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0041.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2022-05/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2022-05/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2022-05/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2023-06.yaml b/configs/cccc/dedupe-para/CC-MAIN-2023-06.yaml new file mode 100644 index 00000000..575513fc --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2023-06.yaml @@ -0,0 +1,71 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0045.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2023-06/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2023-06/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2023-06/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2023-14.yaml b/configs/cccc/dedupe-para/CC-MAIN-2023-14.yaml new file mode 100644 index 00000000..06da82b6 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2023-14.yaml @@ -0,0 +1,68 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0042.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2023-14/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2023-14/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2023-14/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2023-23.yaml b/configs/cccc/dedupe-para/CC-MAIN-2023-23.yaml new file mode 100644 index 00000000..1825fb84 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2023-23.yaml @@ -0,0 +1,68 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0042.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2023-23/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2023-23/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2023-23/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2023-40.yaml b/configs/cccc/dedupe-para/CC-MAIN-2023-40.yaml new file mode 100644 index 00000000..3769dde8 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2023-40.yaml @@ -0,0 +1,26 @@ + +documents: + + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2023-40/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2023-40/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2023-40/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2023-50.yaml b/configs/cccc/dedupe-para/CC-MAIN-2023-50.yaml new file mode 100644 index 00000000..f5f6f19b --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2023-50.yaml @@ -0,0 +1,69 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0043.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2023-50/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2023-50/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2023-50/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2024-10.yaml b/configs/cccc/dedupe-para/CC-MAIN-2024-10.yaml new file mode 100644 index 00000000..f4288856 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2024-10.yaml @@ -0,0 +1,64 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0038.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2024-10/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2024-10/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2024-10/output" diff --git a/configs/cccc/dedupe-para/CC-MAIN-2024-18.yaml b/configs/cccc/dedupe-para/CC-MAIN-2024-18.yaml new file mode 100644 index 00000000..aac2a2b6 --- /dev/null +++ b/configs/cccc/dedupe-para/CC-MAIN-2024-18.yaml @@ -0,0 +1,61 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0035.json.gz + +dedupe: + name: dedupe_para + paragraphs: + attribute_name: dedupe_para + by_ngram: + ngram_length: 20 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-para/cccc-CC-MAIN-2024-18/filter.bloom + read_only: false + estimated_doc_count: 50_000_000_000 + desired_false_positive_rate: 1e-02 + +processes: 188 +work_dir: + input: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2024-18/input" + output: "${oc.env:HOME}/status/dedupe-para-v1/cccc-CC-MAIN-2024-18/output" diff --git a/configs/cccc/dedupe-url/CC-MAIN-2013-20.yaml b/configs/cccc/dedupe-url/CC-MAIN-2013-20.yaml new file mode 100644 index 00000000..0f79bf11 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2013-20.yaml @@ -0,0 +1,337 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696381249/warc/46805c0d8c9fc6af75faa2d92ba0c9de31ea0e0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696381630/warc/bda4ecb19f1d2e1e1139b02c3201a3cda6bcd5a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382185/warc/5bb9daa063a27ec2c05044683bf9a0d32aecccd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382261/warc/0b53c043d81a6c4d331c5de460aad479b34d5c7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382360/warc/20e49e1879eae233f8d42f77b9046b9e28a8f6a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382396/warc/7ccf2df755e7629bb81cae028ed0cec9d67916d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382398/warc/32791d0fab3bba1846160fc7924d7f04596d67cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382450/warc/12fa4ae43179e174395fd2e8644772882cdf62bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382503/warc/697f80c09ceeaac80ccb2928af5f6cec416a6158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382560/warc/14862c09ae0d6bcab6c9bd16eaed35723550f57a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382584/warc/3dd5daf20b67d0381572739190b5b3aae081345b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382705/warc/e59c5cdb46820da52e8487275b88a4dd71a99cb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382764/warc/76301d96618f61a46453753d4b5e48aa5dc96b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382851/warc/837eef8e0a409e2572139030f1a65ecc301324ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382892/warc/bfe49064523bd1a7b91a5d42d4563dfcd9ebf600.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382917/warc/f6097c04db2f886348eae3a901adb3a661fa1be4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382920/warc/2fdb48eb9fadf62ac4a0f1611d8d25171915e79a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382989/warc/829aa936d114953ddc9e526e10ce1464be205d96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383077/warc/588407816cd8c6ef112ca11002fa30b043b69357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383081/warc/09eb6a85cab1f1e31b15361b8208173883b4e778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383156/warc/73ccb45aef841fdbbfd4e3424b5173243b05854b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383160/warc/6a3a4f46ff7173a029ea2b1f87006732d659d791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383218/warc/6e8b6879ad43d66f27cb7905b551605ba9cb6f4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383259/warc/0d60ee4cf5cde9f497a5e3190d87429cd0244433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383263/warc/dfcfeedb25f450e759f50ec0da0fab5110f9166c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383508/warc/dfa8ea0fb40ff878b86e8be5f786ddc556bbfeed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696384181/warc/ae982f3423f26d51f36f6d87fb38766c2507b166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696384213/warc/f85a944fa0b52df084ac1f46c6194eac8bcb7784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696400149/warc/9c25506ba83e6fd7033b78b5e5bf9ba9897bc649.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697232084/warc/49980cd4360675ee58bac843dc98e177d49402eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697380733/warc/0da287c90e344c81bdb2a65c5f608b1c108aa87b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697420704/warc/52cf74cb438faa1c9e9aa9a8b49466c94ee70a86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697442043/warc/998a5ea6b2b3119b3efe122ef733619ccc83af65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697503739/warc/dc01d6563980ba29c36d399cae7b25d8c186e6fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697552127/warc/127c755c6b65e97a01f025d4cdddca2a901dbbb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697681504/warc/9db9b018c69e0ca3db1b22c95da9e14c998f4f4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697745221/warc/011d1a8b3af716cc99d2f309e15255bee8491e79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697772439/warc/3b660b9f97b36826f2c3145be88e9ca5418ad465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697843948/warc/a960f65435f1fc8807fa0324ebe0090c34f553b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697917013/warc/b8759ec2a80796c4893d24c16c7d9159a3b94c85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697974692/warc/6170fcf8c5ace5a92dc0919819e599af896822d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698017611/warc/9fbe35a03cf6a0bc5d08dd50ad2e4590644ff70b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698063918/warc/c00f51d0c5ab6fefbf197487a45f8bc61b578b2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698080772/warc/e4ff76de84939b939de894cd72394979d5aac68e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698090094/warc/85d02ebbfcfe43114066ce65ed516c2a3cdf9987.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698104521/warc/6c09fb5234ff26d717264c9c3f370bd3eca2efaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698141028/warc/f9283cf0e4948fab179d5ad422f03ad2f4bebbe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698150793/warc/ce7d36518318957a97d7a87515a7aca33e41d98f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698196686/warc/0c719e955be656f96ccd7c4f1f12318a2ee0df99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698203920/warc/d2b987ab19d98fa9a8bf9c3523ba36f2b113fda1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698207393/warc/2fa0f8ba4d97387b60bc71ba2c0bbbef823fa64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698222543/warc/77573a4051386b9b54c08d9629dab8e062774ac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698238192/warc/0a4e424c0234c400371ec13f53690814e11ace73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698289937/warc/5f66cc3816b372940f007dc511322b077919765e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698354227/warc/10414f9adee91643c16a4baa9bac6a981d6253b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698411148/warc/19f88d5dafc307b9777bc38084447282e55b8c48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698493317/warc/4650ac43ab46d6bd8191ee330378f3c318386187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698554957/warc/81e4976cb11adec8761e86de329d4ef1cbc222fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698646863/warc/9711e11d786077aa6bf018fb7579e12a729f2fb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698693943/warc/a28a747edf98ec54d3e80374abb48c393118c043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698924319/warc/54d5c1aba654e117f53687198ad40fca5945e6b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698958430/warc/52349e4fee66e33dcffde3067214e9997e2c22c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699036375/warc/34d360c51827de7a2e2b4b946641ca7f6d1f64d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699056351/warc/499a7620b383522235eaa1942fe8f2275b469b94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699068791/warc/cd80ad7a337c76125c7d1a6099ea00fcf348f0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699113041/warc/9ce0842299167f8f4f84424bf1d1773638852b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699138006/warc/80a17bd9cc8c1571906613af5f0f79a325d9fdee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699186520/warc/d58bd87385e120661ce192d7c0eb1a13a5d59273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699201808/warc/407599ecefb153a0968c00bf7b6e3d4f02d7cad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699238089/warc/ab461613e84815296e5535972a944fd93792d1ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699273641/warc/038e4ef959d153cbd51536a9b850d5d174969f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699632815/warc/c09c6c77d6b751322ba498833e668c3a73d0167f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699675907/warc/ede2a52f7209e75793b2a832d715de5764a79b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699684236/warc/e6d3f6e25f69ed26305be3370bd06aecebfeaf20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699730479/warc/72a425b8e2112d655a9dcb5d9006c245eb7463df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699755211/warc/dd1b98e1da1f845f6ffeb655727cea0b7f91331f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699776315/warc/f579ed5039d0fb256166ec85861c80b5d38cf874.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699798457/warc/7cd539e57d1eae519167b40a084bb37d96ed3bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699812416/warc/e9be901b0cb1737385f26b496f22c413bd82e7a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699856050/warc/9774f4091932ff78018c0499cd497d0dfd6e36df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699881956/warc/114f1810414924e3560bbb2b4d9f031253b2815c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699899882/warc/207ea2264258d050e4aca234c71e46e284937520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699924051/warc/7515baf8421ea8bb9329fa00af8bef559d790716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699977678/warc/6a8c9544ed9c046f2e77701d5b32b0344ec73b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700014987/warc/c348fecf5796e0a4983ea98c026d5c7a148c70e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700074077/warc/a9a78f31456541d7393ac8916ad0f5094302dc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700107557/warc/858fe1b8eec0990924f04735cf194810583a9f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700132256/warc/85115016d2a8c707926103df0085008effac0c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700168711/warc/0ac9e2155984d9f807bfcdef60c792188aa30b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700212265/warc/6a0bfc3c4fd59c307caf1ba0567baf7f4d0ce533.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700264179/warc/2ea94751817d62a54570f94215c6cf270a14ddd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700380063/warc/7b99931a4d55d3288cae8a910ae4ea2739614630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700438490/warc/fb58aa83fb652296341e8376fae1e3ea188e9ce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700477029/warc/2ffbc54942ce400169d0c34f7e7d8e782f0d5b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700497024/warc/7797ea9bd1f26e1b2755b8c73c9c14d834f39586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700563008/warc/4d8493e05f786baa437870d2ce8b4219b0ff042d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700626424/warc/b3221b1810053c98e8d2c1f911da5fa89f0a903f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700795821/warc/7725b160bbd9a96142e62daba7e44760b41b3256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700842908/warc/ce8c85c4f4b6bea199807f9bf2fb4d461af786b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700871976/warc/05d7da0e4c0d4ec1471ea60621aec91fb0bf8d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700958435/warc/83c9e1c7917a62c9981476db8a638bd69e7ec6a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700984410/warc/01d522f74fe6a9d5db4b404766ba2d71c640c7dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701063060/warc/66ae4555553458b5ab33a141d1107cb29ff30918.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701153213/warc/3ae64dc6db96db0e8329791d9b0e257b6e53e0bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701233842/warc/b76b23a02692752356b51701afdeda040cc8b197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701281163/warc/4db941df9d6246a47a2089d7acd3435d57f92962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701314683/warc/5c2c2eed56416afab684fe6e41874d77d13743fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701370254/warc/a9053040cb131f5a7112080893c9340734ea8d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701409268/warc/d354f4830d85d027d9dea12ea9c3acff7ff6bb01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701445114/warc/3e73ff0f4bc3d9fe80669b27f7944cabd411c7f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701459211/warc/a23df5008361aeedfe203e1e2dd1e837e6e10aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701508530/warc/648413bb95c56e1422cce5a4a26cf89a68111cce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701543416/warc/ed7e516b96ab22ab605aba247f49d2b75a5c591a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701562534/warc/8581e6328931c7cc07cb1f7308e59fc9a76d18cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701577515/warc/6019f83109d7ed3cf0411e8eeeb872865ece52f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701614932/warc/009f11b70f47eaa6f8a27210f64690c5dda5ab21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701638778/warc/d979eafb0afec22f3248ed1eeef9e56c24b5868f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701670866/warc/4992ab3c9510ce53bd11ebed664d37c1241f5477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701760529/warc/6aba771e0f3f68ecf2aa336d0b2a38ab2feccc6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701806508/warc/4481d3bd53b269684539c162a07dce6880717e1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701852492/warc/7cfedf7527fda75993ecd569e875ed947799ba6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701910820/warc/e88eacfac85db5ab301e487a513c29896ed8f8c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701943764/warc/0cd3833dbd51439a06757644e52a5b26b2b7e0af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702019913/warc/d162c636969d362d67341723d9555409446c31d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702127714/warc/cd118e3bfd443c7de791bf143df7591e868c4084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702185502/warc/59f9df69b9e771d5542001bac54b581a9c555d80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702298845/warc/fee3da7e30f6c00c1c43496ef08c8e7a46e82d4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702414478/warc/f96e8cfe614aac2c197045c62fbb4061e06fbebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702444272/warc/6811866ffc78ae8c348fcf939f5032534bacffb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702447607/warc/9f61ddeb3f80ecdff5058fa3e57d0162a7833166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702448584/warc/a77bd48f436982e2c2e6dfbb377a166448e1223c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702452567/warc/5c11836b15b266a315d53fe9d9be6ab9b60195cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702454815/warc/8fe59b523f67cedb497d417c247c26016523486e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702525329/warc/782dfbd74f645b26c1e99304bd90b6768171b2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702652631/warc/cd2e509ec5ef7771756481a428a12e429a00b734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702718570/warc/1e5fb62730fc31734f63aa5bbd4236492777245f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702730377/warc/8e202ede54fe538a5df5c3938abeed47131881b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702749808/warc/5211f6b8eace635fbe3f66e2df34880a0880b2f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702762497/warc/0d337ad3ff6dbf5fb9beaabeff88b47874677d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702777399/warc/b7912b1d191468800bec03b0b215aedcf5054ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702810651/warc/7e68d90d7d4f28262d9f9bfe7bd5862385c26f82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702849682/warc/58cba7251546e59a2cfd5334a81e83f4a761a97c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702900179/warc/adc404ace4e532b68eac2f696d675c921af2c181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702957608/warc/1fe269989d23f9e29aa5205668252ca463b32729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703001356/warc/a9c8565a1eb43b62e73d4f74a6f052cebce534cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703035278/warc/58561de696a4ba1d6011d2e7f66ca35843fe7837.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703057881/warc/243dcd8b7bbe37861ca50c5651f8134a7e9ee84d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703108201/warc/33ed1ad4403db9c8f825a1a080638425be086072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703227943/warc/5b5cd01cff4ef51e8257e6e60e135badeaa4cff2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703293367/warc/882b765626e78d5567b908265818a1e26c84a5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703298047/warc/54dadb1c1ec1c40d11a90c2c218c828cd1405d02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703306113/warc/fe2037cf4114fd48854f711d8f9b92f733f2da3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703317384/warc/5a16697086b21b04a894299ef35b0e8f9cd3050b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703326861/warc/501372d56c63e6df04256df59e32d85c45931971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703334458/warc/91051b450c860b0f150e0e2e80537075e33f05f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703489876/warc/92309cc5105d7f1afede6138ac476e27031aaf69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703532372/warc/36407e0699534ab1c8485abee0acc5e04eb85ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703592489/warc/7e617741ce811683ac9c64549d35646da6e19774.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703635016/warc/dddb97a78b383fc2deab5c0aae7572639282202a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703662159/warc/93fe12c4a3052e28bc4c6a06bcde1db70862e961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703682988/warc/a836fd16d7d2ee63ad7c5f55d24b2d1a12e0b825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703728865/warc/2b2c9fb05e69a147282d23467bd8039747e9d6bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703748374/warc/98d5e46b2067c8213e3590c576f151d17b9fe710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703788336/warc/c7f1e22922b4147b6055e0275e17e6a4f1bd024f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703830643/warc/4677f89f699039c83be6469cef03cc4a2220af95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704007597/warc/7e44962d44dfc43cdfc6053a558fdfc96b85b943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704054863/warc/8d181c34fe46c056a004c6013d9227f319747687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704075359/warc/8b2c60f8a6752597ccfb9df3c296998f3239a8c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704117624/warc/966b33abe46e20872830af0bfd898e298737fdc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704131463/warc/23f0fe872a7fc2236cfc9f59050e83e264098cf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704132298/warc/2084b7dea09bbb735865c7f701a24af92f8ac069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704132729/warc/151f83c2331231e7cc875647a8a3aeb954bd64b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704133142/warc/68dc4706cee08f92d71d4427433a7b91acc5c1d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704134547/warc/c8023cbfe734462d222276a4d0a6826a62ab9613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704179963/warc/145d71e7f2a8fa37eca9f5342b7e6b6778cd1697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704218408/warc/2cad967fe2ddb0de41f5470e9e9d7f5550a39fce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704234586/warc/2407faa96f7182f274aa5a7af5313e5c473d26ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704253666/warc/85a76ae1e4c92a9b9050b1d5e9139c5ece0960cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704288823/warc/6cf59c270c7d58cd8993a64c8f9032f341853d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704368465/warc/85e21737b7cacd06c7192035a2310d7aec9c7f85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704392896/warc/b638e26435d0de0459ce1dcbf3b45a78bc6d4078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704433753/warc/d557dfd3e46cc99faaf500d97f871838e3fbcc1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704517601/warc/72a61632834bad28d1abb9d1ee79054cc11d0948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704590423/warc/09cc875a4e64b55c37c20fea7e76204dec99c697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704645477/warc/42eaf8a62e76562172a13c1820b9707a385db40d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704655626/warc/a07bd2d068d9ac3d4517ccade51f566d203aed33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704658856/warc/dd6812ac1739fd9bf5dc234486b75074227f6f38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704662229/warc/3b139b0cbf670ce42aaa8cfdc692a79fb447cb6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704664826/warc/48f4de5803261cb3e2ac174f5665365c65a3adb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704666482/warc/9d7ca7223b50745165c3efde923904039f51a76e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704713110/warc/589143f0a50045f501878055621ce7790d0abe8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704752145/warc/5ef5d4a2ddc5d01eb979c9142d5124cd0b449b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704818711/warc/5c9b06baaf24aa9d805e0791ffaa95de89b43281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704933573/warc/2f2f1cb9a67057cc7a8a8f70812d6d53e68ff64f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704943681/warc/4763916967ec95c93dc710ad45cceef0f50122c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704986352/warc/4a5920231d87fe386a56f07fba672f3e1dbfba21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705020058/warc/aa52d25a7992aaddc002d915a82cd9b42f6de1c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705043997/warc/c81941b1d567ae7c4519650f2bc2b27d0151e816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705069221/warc/692cfd9240cea46a032f3038ede7dc0deb25a74f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705097259/warc/bb0e2f60e170384d6a78ee017adb616c56287ff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705195219/warc/f85580f4c680cb7f2ea1ee9f7b849367a0c9a0f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705284037/warc/2143c283c80fc3e6b6d638ae75394ba3cd70faa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705300740/warc/450cbc5eb5385158041acf0eca2464ca04e37105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705305291/warc/9f7aeb5c6433c3dedc5eaadf23279bf18c2f1d1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705310619/warc/fc92b7499384d0f866ebebd2add0e9545ae12728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705318091/warc/7ff4fdf9965290d48ad65c205df295384091ae1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705352205/warc/1f06ab6218b0b31ccee526a66ace8c8f6b37caa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705407338/warc/f6ee0fca6196fdb2741cc379849ed83a76e47835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705502703/warc/b1690f99c1734c5f053ff5ce21cb0e4e8835f85a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705543116/warc/c931f8b271a946c6366538c76dbce6e039ee84e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705559639/warc/414b1ea2263128bd622e13861a0fe0d727f7f151.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705575935/warc/595746f8bf1659a969d69fda4d9bc670d3ef24f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705618968/warc/c3ea49fc12d239003b41413775af93304fb43df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705749262/warc/475cb2ad39365186620f9f88e6dbc9e4fd0bb88e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705790741/warc/254fb7d45b4eac3731a53d16a76db433830dfe11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705884968/warc/202de46ccb521b13f9d0f02747dc760567e1d255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705926946/warc/b0c1683e95e7f4065a7e66624e4461ed03a7d861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705936437/warc/0b539352f920e1a5554743883a37cfcc2faf9dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705939136/warc/9b6039552ff29117d0f4359f55b5c65815d1e1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705948348/warc/486acb0854110aedafb542093df80ed2785218a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705953421/warc/aae1eabe04ef43653738c771b96e15925fe29af8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705955434/warc/9e83a37c20d6d450a99a9844545198c05ec08d87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705956263/warc/420816c38c8c51e4e9153eccdbad50e07e4da792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705956734/warc/7befb1ea00c1029168ab41da8031780de78bd18a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705957380/warc/d1afc7f974660d86ffd482fca084ddbd914518fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705958528/warc/e37a26dd28f000d762272803d89fafa0a1a51b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705976722/warc/9a1b9bc786f2c34220027198a20d40a8a347973b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706009988/warc/111e7341fac411b1a10ab4901583e1e7f6ac1761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706082529/warc/12078ba314fd3a34c97449396bc91b118624fdc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706121989/warc/953f4e0b1abcdc3e8971a416ca7ffe7e81290ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706153698/warc/f3400d85e7de5bc2ba8739a7ee853d1497cc9e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706298270/warc/42357ad0b4cf48f551fac2b5a2c1a7e5acd798fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706413448/warc/5e6ebfa6837b0694207ffeb14eccb51cc2a9dc4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706469149/warc/6c7af46cff19646fe48c28a63d85ee4c63d47293.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706470197/warc/3e7ed188565f550c762489001305e2568dab3b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706470784/warc/ce5e388cadf7f6882a2155838849cf3ccee618f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706472050/warc/c4ea545c8367dcf3c200df0dd4b59863c9109920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706474776/warc/fcaab14b5dc20957b725f5a624c90825f80578f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706477730/warc/6088377052495a624381da697ec7ca81b0099048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706484194/warc/31bfc4673943f742fb3df5509af4069beb59354f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706499548/warc/cd68405c692dd65a384fc48164c640a38dd510a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706578727/warc/a76df54fb7b59a78be8a5d71c2b695cd737e5f9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706624988/warc/1a83bf3ebc244ddd3cff36dddf8b776eb5b28299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706628306/warc/7a2df1d4be5adbe1de9960ca7ccbecb6522203ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706631378/warc/3dbd41f8db8bd75649348f9a7a7eb0f8f1b02141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706635063/warc/e52aa8d56c753598e4826193d2a19990cdeeac4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706635944/warc/f6a0f9b5ef67ea2533539fa9a11ce4e5eeab6a8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706637439/warc/7cdb7fc237afea9614f7f23cd815604b39181204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706762669/warc/dbb7f35b0dbaf291ad04a3035f1d1aeb36f67e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706794379/warc/0761b1acd3021ff69fe6ea4324df043bd00d7a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706890813/warc/3892fc08848c276318348d6186d55d41a73c9842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706933615/warc/bc378cb0f083c07bfd5468e0a3e4dfdb731e6790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706934574/warc/6805ceabc0851a7c6119659a1e09e59dabdaa771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706961352/warc/b3f83dcdca295bd4b403f1749608bfd5092d11b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706964363/warc/382a5bd170d31b1ec5bde03f6534a52755b795c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707184996/warc/d211079b25da4736494915cfeff7324ccbf7ad29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707186142/warc/5b338589da756bf943692a90c483623cca0d51e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707187122/warc/495784012d637d2444616dbcec5f0ddb87ed4559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707188217/warc/fe3e5b7faf7903b3a30dc661dd949b9898b01631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707434477/warc/c0bf5a23ddcca991873ec3eaf2f7deb6a4d475a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707435344/warc/ec2f3f481cdf5d5b41339f8dc113f90b74814dc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707436332/warc/dc3fdf6aea2ff0e30282b44c501e5867d8b21f4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707436824/warc/7edccdad3b34aa1dc0cba52c0ca5f786298b224c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707437545/warc/fbc00c6111711d0145e43ffe951328ec456d7b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707439012/warc/6ed0cad7375e2658bc04b46f2ca9926328ee5061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707439689/warc/45a331cdb7f6df7c53ec24b9a86b06385aceb707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707440258/warc/cc7c32de0eeb4367602c9eae7bd181ce0e8afa6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707440693/warc/3e8cd19edb2f32a1ebfb66d333655df00afeee81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707773051/warc/7b4780bb0186422181b32f93e68708b94551de36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707906397/warc/0395304dd56c04d84b885d01c09fcf72fdfeaa4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708142388/warc/5350e2b48ad07e21e843e0f5b2d7855dc56e31cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708142617/warc/f0c9101e728ae84d6608e4b6891563f725736c5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708143620/warc/fd0c113c6c7752b91377e674960b4c309f77a253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708144156/warc/a1cfd4c2a0be7306ff46774967767ed801452c99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708145189/warc/ba6d431ea04bee51c263b49e927ffe0d8c8941b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708546926/warc/b9b8a9e3642f54f9f745fe0308ca68093fc1f914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708664942/warc/562b9aeb7ad1be65eff632d60e0becf5d322aca0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708690512/warc/a92c85cb4f43c5a74c653e57c942da5d9324a5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708711794/warc/fa6c3e81302cf17bbdfc4e5b4ce5da0258d26bac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708739983/warc/69402ad664586c92b22b8334cae366e579fe0923.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708766848/warc/b1800cf45471a8645a7fabcdd306e2c260e095d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708783242/warc/af1e45d2c7b9da45f47d9257b90aa442522ddd52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708789647/warc/1a4c724aafb951cda6979d59d7983857b7620657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708808740/warc/ccaf424d1ccff7d204eea460d8f31394993a3c7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708808767/warc/3aa2ef24d85d88ffe206dab7e4c558b67c1158fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708835190/warc/16ac20f74d3b0818c3376b2dbd93dcaa1dac0e5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708882773/warc/97a44b4541b59d805a077c9af241bc8685973b58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708946676/warc/8c6907d1c0a79bd2a9463b22591fbfbd3ca7593e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709000375/warc/5d5773fe87ad68f7dfaa567024582d4ff293d22f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709006458/warc/34ee9a1749eccafe8a409b4c05a18f425a3dac0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709037764/warc/22ea0208468ba30b8ade09ffdda6859aa7f5c994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709101476/warc/a054235bdb77ade276880a55436abe233ea1b607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709135115/warc/4d88079f3beb72370e26bab9cb63ef660d34d5d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709224828/warc/1664226b192abd72eeb53f4de19c5e3c276bdb19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709337609/warc/c3e473e283661c2c6aceca9ac5c6995b27498d81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709379061/warc/54d24dea88ce85d8a84b0d6b80578b84b38ac431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709805610/warc/768ae88c53fb8b941e330c4a0243a6fcee173132.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709906749/warc/2c67d5920c4eb9fdded98d2eb0ded234332e4407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709947846/warc/7885bff677ea1a101fc58883f827a65d9d07aef2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710006573/warc/46a2ebaa0052fe6e68a3eb883a1f5f1722b06321.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710006682/warc/db3e497ed64b113e88ac8dbd05ced1e5ca966aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710115542/warc/040d64995cf5b65c878555c29b235b8b4917abe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710196013/warc/6ee8ac48aa024989ab3d5ecd4567772ad2fdd077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710274484/warc/437b3a3cd08b3e09f656e823361c785e2f6c4801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710299158/warc/38a1844ca58c47ebbf96a93a2f16ad1759e02cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710313659/warc/8bbeeeb81195cfd8b56d8096e24f956dab21a0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710366143/warc/d07fbf9f77cbd3a7f9590f244f3f6cd37c9f71ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710605589/warc/94987cd17f91899de5a4d2f4b326fa46e67ce280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710963930/warc/16ebb1a1020a7d2358a683945868f1c9484f7bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711005723/warc/286e93b149ea8901399b5f38b447e9e231d2532e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711005985/warc/86c592625f236155f2c23c462a40883cd99e5f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711240143/warc/75d179a045962431d1895a610175ff5d33c1dcff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711406217/warc/75e08cebc7a15793dfa34b7107b0855df0019f35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711441609/warc/428272d463b941ddd78308df34aed2997deebce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711515185/warc/c7fb04c1977fd4e3aceb4e9d392bacb0de122019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711605892/warc/27d25e67c1fe8cf2f40abf8d93eace45368789ce.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2013-20/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2013-20/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2013-20/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2013-48.yaml b/configs/cccc/dedupe-url/CC-MAIN-2013-48.yaml new file mode 100644 index 00000000..fba1bfbe --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2013-48.yaml @@ -0,0 +1,540 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163035819/warc/f07dd650605c6b8f5639a3b0817ccd81f24bfae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163036037/warc/a8c517a335f1566d07c695b0b8a11837cc9d643c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037167/warc/70b113a6fc1c2f5ad722060abe1eb6591e8f2a27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037418/warc/733b4f54f663fbde8bc05aad368414c4e1aad815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037568/warc/e13a0e4f8c78a1f303ac97f310e8a451e9eb1a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037829/warc/dcc06034155ed5d5736d48ebaba39f671af4ab1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037851/warc/7734b39ecd1d526a5878a2ed60c1d3a6cf10c482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037893/warc/fb41af0d40b7e4c34d29f9e20c409d5c268cd245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037902/warc/6e59be77be5aeba67e78c6450bf063e68ff9c0b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037903/warc/58d1ae906271cf7f019806f279dd9e5701ab95a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037952/warc/27b99e91be772d371e019c3ac3a69e0f73a736e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163038079/warc/fe8571664db635080c145397d0f883072461ac9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163038307/warc/596c5c544041c226fbcb599e2b45b81606026e5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163038799/warc/c50582e166f135ba78f4ae0313d37d10f03b8f60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163039002/warc/14564be23b06aa78481a3d8cc0029b156312d229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163039753/warc/5abad53915cd4f7697c6b766a8eff7269f9793b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163039773/warc/60fd98f45e92cfa9df6226d1560cf85cbf8f152a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163040002/warc/fdb7f7cd953b46db6334ced2ae4c00d82856b8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163040022/warc/845a6e533d228484c65be6db1b11064afc1d8263.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163040059/warc/2771bf3d0f17dc48867836d4f0b0ab93c016c2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163040130/warc/3ff7d1e9a22fc3ed4f8013a9ff067989afbe8f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163040712/warc/6e9cd04114c4e7779c2a7facc520570b1f47338e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163041297/warc/636002bdcfce22947faf5590cc4b4fe511831db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163041301/warc/64bad7341e09907cf7f81e2081252d022c6670f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163041478/warc/743d87d2b55932efc3d2309eb4e5c9c0c586bb09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163041955/warc/67997ce923125443f913cd8fa31fec454cd5476a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163042403/warc/3edf9f2095d96f5b27bdcec96b66419ad6172fb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163042430/warc/468556c0bb03a5dc402ea5347819e13f9bf92d15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163043081/warc/28cd91a800c56194bd3156ca9e3fafb37856c3bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163043224/warc/cd4743028d966094b26e505063a97ee1cd2266dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163043499/warc/4fcf08b20b5ab2bdf2d5b93f31026cafb6b99bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163044233/warc/4fca9f1854d3f4fd71530e1bc7fb68eb8fbe25e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163044331/warc/7c93180ceaaf891f05d42058a6efbc2d9b3775eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163044524/warc/37fa8bd4bcb1d6dd4cae44c5dc5c46ec62873b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163045140/warc/7c9047975a547676ac4d52dbfef08b0584a1e51b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163045148/warc/23154fc127a307ac42da56154bcefd2cc6045896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163045217/warc/a2df5d335a52cd32d258e87b3f651f18ed9bc334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046049/warc/4222c1729c291a88bd1523628888feaca8d4ff7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046151/warc/dde7fca58d188e147f31ab4294b61e0c88077e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046758/warc/cb78dc426ea7c47c2bb1c30c5707a5714c09d156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046759/warc/d69c20f3ffb364b99efcf5bf8a224f2120c9e892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046799/warc/333844e7dfc187bd643a54ea3fac4c8a28327932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046801/warc/d601c4ee358707c063421f7bb2809e20874a77ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046947/warc/483250c328da26cc6020319860f261b9aca91373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046950/warc/82898610bf7991d89cff157a204a11036f47362d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046954/warc/455e8c215fb0f93a3ac0ccd5ee2c1dc83e068604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047052/warc/a901481b06b612c62d89afeb64e32767e12992d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047055/warc/90af7c2ce88e7eee34926f2295f0d0639c2a6c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047212/warc/7357dd1161709df4624f4000912505a893455006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047214/warc/9a6978be58aa4ad9a93aad5aca2a31c2cbdd3b4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047523/warc/5e7484a5158c104c60eee2818bf63155d66b42e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047545/warc/9d75e5ab66591a4a28cb9d0b31467bec9d6599c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047675/warc/f78b5b4ec69117f6d6993b451f8d7244ad3272e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048129/warc/a5c41f990e83507c4f2b27cc295954a6e15b0c67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048362/warc/947c804069a8dc09c8bb0d03fb92aa793e0ea94d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048447/warc/fd51c6050bfe4baf4c8015f00a5e85c419a511b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048614/warc/b844f58cb1b18fcd12b2f42137ebc74fbcc74a2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048663/warc/594b8147ba3bcd87b8cd7698236f6c5205d86a7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048688/warc/d90a789e4f4484ff94b8ebf8475534b964087a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048803/warc/8ae71ae6b586f15ee700f9afea53aee6a0acef4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048970/warc/4d62eedb1f1b6df09caa1f91668ae26f3f4e0ae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049020/warc/c7445ce0bf82fe0c9154318e4ad03979d1804337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049340/warc/f2b56490ce11af4ea9192b507f861f2f9d8bbad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049455/warc/46b3d22d82b139c6f62455ae77eae03c6b898056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049486/warc/80e0944262c4627d98a2774c5a8f903bd5fcc738.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049570/warc/7a831e13b7e124626ac58d8bcc243f7192722654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049608/warc/88819a6f317da8c2ffafb4e4dffc6f518c9e8c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049615/warc/0c7760b35457550dfefba641030e021a019d2bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049631/warc/fd5366abb47e8c0136b36ed6aa8a1c292420a3ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049635/warc/cde9d9d169c518613780a5028be73702c192aa65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049948/warc/be397cffa4e4dfabc40ec08db2fdb185c822527e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049967/warc/0eb2b7359415914aaacc2bd2ad3288b7e8833250.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163050081/warc/f3c6ae6e215c587a0089ea80e54c4f236f030281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163050122/warc/01088117b0f0360dec2d71161b33d1870fa7ec6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163050500/warc/0b6822c487638916921c9c7990ccdfabe89fcede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051139/warc/de1d05210cd77da15721db25a1ba450cdfd995fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051140/warc/c96920af7185eda81759348865403a68af9ef105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051244/warc/938e2753e7a6c4cb7ce807ba73b7e4be99189f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051248/warc/91231849b33951f34aeeb3e2ff34b44bc1cf465f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051474/warc/f2758adaca47dcbc73131f226bf4946bc25a8c6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051476/warc/d6dd18af9a3de21a8c916172666aed62a2ee8f64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051509/warc/70a33c9b635716042d83661fe55ba1b449963d29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051516/warc/3f28faba028824dec71387094830d51780393791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051588/warc/0bf3a00588a867e137ab7aa2f52cce32aadc9b92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051590/warc/2abad996072a925236316db38fb2a63c63ef0905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051684/warc/3b6b22dba325acb07d50e172bff739ade7b40aed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051776/warc/4df8a9f8173d09a88ccdbef6a6bbe2353b324804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051789/warc/46a7898eb19f19b57efcdd33c30b8f2dc70b8462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051984/warc/c88d6fd2592138136d1d343be04f6aa55b14e525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051986/warc/0e9844106667987b441b56f0799e39a97170a414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051992/warc/56633bd5ca649bd068fb159dd406aec005c7c5e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052034/warc/ef7f97b8c1ab442817b59fbc4b7452f2291e353c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052107/warc/d6b197fc998776c4d26da06b3619b41a6677245a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052204/warc/f499443ade697b12dd393041328eb82c32ae8741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052216/warc/34637d1004be441127b8a8f75768bac91293d850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052275/warc/5fe6c2b5d0b94f14f8a1ff8b43918f5e18674cc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052286/warc/70608c02e0d78a5fc4bc31e41c3f7b14ee62fc2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052338/warc/eec80967fd84d67a6217d1c3b2c535ac43efc0d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052343/warc/a8b83873a2e52ec4e04adcfb7001ead7768ff1f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052382/warc/d67df5c81ae2d634599f438ec41d1abb5ae02aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052462/warc/75f95f478442cabf8375f9d5e3cdaee1c798e672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052469/warc/4e5ea969023981932fdfd4f76fa957c7868e3cd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052537/warc/fd893942dad0d09f342b3adca51ef067fe4139f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052593/warc/3b53bbcee1cefa9447a56fbd05aa19dbbed2f6ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052641/warc/ab015eb6e4657282a8f4c020a5f0842671ea0330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052712/warc/d68d18d369c43849b7ef33f0094bf77441df3e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052713/warc/bf4905a9c84c3c729c41bb3e16efdb4d9b744900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052727/warc/129feb738ef7dda0a53ddfbf953de0b4856d0cdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052810/warc/6faba4d4b1fc5d7bc00b538969ac6d3bb0a03c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052909/warc/e9e54b2ed77efedb2bc91b9ff135572dbcfe87c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052912/warc/12f993a976f1945d4ac44c37f9cb2d4e6847bcb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052949/warc/9ec58199a0d3bd7d33e3e2626dada60ab8bbb6e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052970/warc/5b3760965a269adacdc92f7e71c1449c0e13cf76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052995/warc/370ef37f395ae0794bc21c29524fb013011ce0a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053003/warc/1d565322eb38d4318de801430b8aa205c2e787e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053174/warc/f7558772fb0663bec8f416465e78824f2ef317d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053330/warc/af7d306fa40e88c6ecc7bc6f3e2fad2c5b02a445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053380/warc/89397262e8064f1128e84b02d6daee4fe79b80d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053558/warc/d0f0a57641eef951dbf7661e2e7f85ac09051285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053578/warc/c4be1a652fb619da2d87a4edb9707c0ee71fcb23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053608/warc/c5a260894c9882dbcbdac56f78f394d059a11c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053664/warc/c0a9313c4965995b19a54ef8e2ebfe7b256b4896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053669/warc/9d5bd88d940ccc0462ae67fab420aab202a41bb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053831/warc/bcf3bc3c8f7548a81d6187c67cdd0caeb3d55221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053843/warc/4f7cc161d63eb4a4bcb783de2060eee0860165d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053865/warc/192d078df33824979542d3cb4cf944bfaacd7be1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053883/warc/50ac9af53aad7fe6ab961a69ef2fe37a859d89ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053894/warc/a1b6f4cbcbcaad25cb3ba80f937662454dafa7c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053921/warc/30906d7a591f08ee2071f296ab925de9db221cf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053923/warc/5cfb0608f40b4677fac9836f85f29a9ddf529d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054000/warc/ed31bca3907ae9e4b5c2dac57cfb9871cdda8f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054096/warc/9c6895eec4d068e93b760b2574182a0aac1c809b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054352/warc/8a1efa0bed95d06638ab3223e39cf68356e5ed6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054353/warc/43dc55c8aa7960505255dd3113d8694895561fc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054424/warc/e25d08454115d48e53de1048a8b05c307bd28f06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054457/warc/d4315ea55fddcbf203f927b2efc3f8fcc451310f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054548/warc/76489d521e6fc8c7991d83355500c7656ff44859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054576/warc/43a48abc76c73b19645d779c8d4fb9c4e35bf7d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054599/warc/180fabdbe6b608b5172bd31c83f70d6266313fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054610/warc/27ae1c0d1192d37dd51cd38e59c3928c132a3dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054867/warc/bc2fd49061ad887bdff2d5e57b705c86fef636c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054973/warc/17957aa7005645ac5dd69b96c36239c8e3c3517b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054974/warc/1309102eb7b785c28bb92c36950ad99436a5e3cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054976/warc/2ec64939f1826b8f9bb9f4166030d30a1ce532c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055217/warc/54b53abd8838d88fd7c4f3a4836f1d095f370db2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055516/warc/5a071282470723fc566bedaf8adb566160dd75ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055633/warc/10c34c8a15bf1883c3ec91bb5de27f7fc48e3e78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055701/warc/8211f9665e3d1d7b322bf8eaeee710b10bc245f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055759/warc/e8a81d7436087e1cb895df04f07d109d0cd5388e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055782/warc/1e49e2c16f53a7f22336470e2f9c3c44cb3e0721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055810/warc/4efbd76463bb5e15cb320b336396cd150dd4350c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055852/warc/2bd6053d5c85b5874aea1233ece1d16431e715d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055855/warc/107d12a97efa4945790a2c6d9a0fa03c1aadeace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055862/warc/503364c453a0766da95a671753829c3c4f4c8662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163056101/warc/5f014a7ad9d14e2bb582a00ade97b3c9438e200e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163056120/warc/8fc48770bce83ba32ab5afc0d7af4c3276723c37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163056670/warc/91cdd980356d628bafbd1a0d5d4e0ad31dbc62f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163056995/warc/aa52e4da78c9914e78387147d20913127881a7f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163057146/warc/ebef1ffd88a8a6d9c19d9b44ba9569a6a9ba3f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163057372/warc/12efe0a68f731559eca1ce2a6b59ba02939f553a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163059081/warc/3e7b6aa74941f89a22ed5df324caf67ef0ad28f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163064915/warc/47ff41771efae6116116a8e022f141d25f2694b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065002/warc/9e3775aebf0270793506b492ae615d8b0f364020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065046/warc/960312e8bba4d23940159aa86e655d20e5772ad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065206/warc/87f8dd97eea4f2363ea5fdd1667c5e42c09a782a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065342/warc/176014fd29539c581015bbde730fd665f5bf1879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065409/warc/746dfb577a64100729575ffe508741919f2bedad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065688/warc/8c701aac09dc0b2532e25d9c4f015df68df0632a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065790/warc/78e83f452cc90022da09c10eca7c9ae2b8a195ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065834/warc/6c2e4ed5ed10e72aa73ab569a22efac59cafc126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065880/warc/002c19d0982dc5f9fb32f19ca3ebd96b5939c759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065934/warc/b703eb9c29559f0f783163e987ac59b8447e3d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163066051/warc/8f9a42ec263243949f22001085505effeb13ce94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163066095/warc/d94778816ab61bd43b17867896ad0c8699077666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163066152/warc/a36877df0c438a50e84cb609b436d50babdd6173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163066263/warc/893ca1c8e4a6958a10253dbeb75c3b4def6e2eae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163066444/warc/e86b40319a19460719766aa05606d03b0b3c6b74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163069032/warc/cd8e278df73344a03d7f1e6737bc922c29b7ecf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163728657/warc/eb2851c39911a5570601f57bdc99ce66a11b6210.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163732500/warc/6d013eb3b1eb2c5e0a8c18a0e3da6a8830d0e725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163785316/warc/1efa879974361456f9dcac968e36520f71f71a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163791972/warc/20da8eb50e716144b2a63e192d4e3af673bf8740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163798888/warc/82c199534afa21976c24c69fa9756d3911a338a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163800358/warc/a7b5506d2225222608057055dcd605a058e643ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163806278/warc/d598f873e28e8efd5c1085b2584033387dc58fdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163809293/warc/4d7353b8942546d09707ad8d01c9a91ab63cd212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163811461/warc/f9268b961c5b8b08fdec752681c19c01252191a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163816314/warc/f64c15489fefce9b6d3cc4f7b7f40965d3e70336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163818502/warc/9349fe4149b1dc30117aec54986ad33aa968343c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163824647/warc/948fc35c6ebcb2d87595002a9e4232c2ae856022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163826391/warc/738a048ceb5f4ad56ff6d897867902611d970229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163828351/warc/c8719579536ec95971d58d8cc4f82041b5885d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163835370/warc/88232045c3893b6a838056c595cf94cc5592c267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163837349/warc/240f12a7cba63164f0b12ec350d25753191f0dae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163837672/warc/1e6b22d0be46d91df0006aa3a602ea820e928b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163839270/warc/7445e9146ea70b465b8e1ee3ba85655763ff2187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163844441/warc/eb293f4d362502720c5632b6c06322ff17e572b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163846391/warc/270c088f24268a6c12493d17bea568174bc6486e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163848048/warc/3113dac581204b20388adfcd5a666f762ed98415.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163857457/warc/eff951579eacfafd61b7530659927c058fced360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163857566/warc/1ee268ef2dbeb1d3006602493ff935ccfb1fa7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163860676/warc/b22e6bb3028ab6f6e89ffae4f643290102270451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163870408/warc/6617da611db49eeb16a6283d024e8ef8cb310d44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163879441/warc/33cec436cc5c5d930411ac6d44a4fde1ae898bbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163890759/warc/e558a5d101e10d184ea0b743cf9b67e12f1f7181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163901500/warc/5ae488bf35f6d9b018b84332128aaf828251c9c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163906438/warc/6357108dc20c8c6e44433ead22550820895871e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163915534/warc/123250839ebeea48214af779e0409301b883e78e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163922753/warc/b4850cfa690d0f7493f6063160bc424984eff7c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163930735/warc/3a8b6a3917722854b73acef0566a054c6e021f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163932627/warc/cc0cb4cbd7bbb56f4d5248461bed86c3700db8e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163933724/warc/89b1e267323176143e656fe8dd22deeea0ac9af8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163936569/warc/740d23979281501547c61c994373963e70edefc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163944066/warc/bd65fb979bd672617a77a47e9e86a6b0b72c89e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163949658/warc/d617235c56015fe810f2cc2f76e7602f06df8228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163952819/warc/9e0645bd83b48174ca2517ad121162ea7621b78c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163954634/warc/823e1d48a82d83be45445ef5ddd7484a6e3eba28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163955638/warc/545374562426fee2b177fb87d61533f65c50003c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163956743/warc/4247edbdc2b6376c9427b417a10a02b606ce795d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163961998/warc/89b1b0f2c3abeda5b7c1fbe02d4e43f07b21c856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163964642/warc/ee59e69e00b9507de907cb1aeec95f58556d30af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163966854/warc/5156f710a47524077c417719dc83b0c157ed360f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163968717/warc/8ccad26a045c694f102cfbaea68d43cdb7080b59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163971635/warc/9d406166336f7a455ffe92e4dfb5c133c1bec772.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163972679/warc/34175ca2a888cd72c7ff93d93e91da2445acb4ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163973624/warc/8c596ec26fb263cf812c6a0acb4c9a6205087f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163976781/warc/9bfa424d94eec979a9fe84ceb57ff41b243fbcd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163977808/warc/10feeffa5b35ee563eb319dae686983ec2cdf355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163982738/warc/b4132f9d0acbc2032d403f6724991ad35c5f536d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163983709/warc/84074a21c60faa003678e457499fd0340ec42f45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163985723/warc/145c2f102249f29f636c3f80b34128fcae3ee196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163986869/warc/3d3a664c92a82ee78fe1ac4d9546a51c1f9455c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163988740/warc/a595cc525e7ba4612d225bff8a9c54149ca1cd42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163990831/warc/5bfc668f3a7d9145b1fe380ebac2449edbae6225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163990989/warc/7b5199662d3ab8144d843a3fca4398998a76cd48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163992191/warc/39a73ab46cf9c1d33185da1ec4637c2652ca3aee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163992799/warc/b0c1dd1898c602d899490999e59dbebf3eabd0af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163994706/warc/80f73e6a147ba9230c594542d6e47686e7b469eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163994768/warc/844143beabaf8c81953e144b632f14a9d173a3d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163995757/warc/8e3b7091573287234f72b0907e0d4ae3d50b52d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163996785/warc/dc787b14cd724935c07e33abcaf1047bc0990e69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163996875/warc/2a6835ef24a6a0d3f228f10b106d4f65eaef5841.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163997135/warc/a522d4b699e35130011dc37a5c7fa534913c14b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163997905/warc/e84f910e19e0a419e6b43bb63692291c4a9a133c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163998145/warc/5bf775a03fa4f696de9a1c249f0b54872e00a495.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163998951/warc/ed4baa17181b0514dc1e6d1a09026fd155f74fb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163999838/warc/29bb4cf551bc0b33c9d6c54e76a852374cd57da2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164000828/warc/48dd256b487034f13a4e12e16038012583d6ddb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164000853/warc/2438fc4c1bdf2db39d0017dd95e66544fe5a7299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164000905/warc/eb2e7086cc8ae2a915baa1a86fcd6f27cf4bcc88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164001281/warc/d72e97a73a5ab580a3ed368b7af9960edfb0c509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164001959/warc/3f84af50ed28ad9a89094ea8875b3993e979ea2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164002922/warc/a8f2cd6d485beea171fbcda48825ff5115b5e4f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164003787/warc/1bc90e2b052696b1cb9193eb09ef63c9b8ebf3f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164003799/warc/63aea4a275a013f91a4cd7f645939706e78f50de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164004057/warc/f00d4d2925a2c59d03e8c9d906df87e6fbb43f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164004837/warc/2c4c6195362fbaaa117b28cdae74e8a62a464390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164004946/warc/19415e1cc4e59808de5dc727473b173d98d8b97b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164005827/warc/97edfeefaf1033ad4e97046d855a25eeea8d0096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164006791/warc/6a5e1d8b7850d846eceea08fd5762682d11bd5e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164006951/warc/84a74156e202172e1a2a7189c5e1eaec326ed609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164007111/warc/6e6f60f60b3af358899819ea9802581bfcdfd6d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164007955/warc/99bc717ae7c1c39c470af3e5af10c4472ad06056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164009403/warc/8660903fe3b10f655bc00542a171cadada0eb1eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164009872/warc/e80d5dfda563b5616ba6a32a517e7d3934c74e0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164009894/warc/90d49b45efa837677df0e49991ef23538b5de914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164010865/warc/735437805ac2e0d6eb9cf7259fdcda2bfcb13cbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164011314/warc/44872846fdf18689e3663c8751fc67bc2b8e0487.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164011870/warc/d62f6587700219304d04d96aca67a376d0e11a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164012753/warc/b8ce74eff793b42f41da8e8ad2f254eed1312370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164013027/warc/7a201b516273ee95da793cd505a3d40ce69b08de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164013918/warc/e72137784fa42c09ac8b5c1773c5084e4a67d7b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164014017/warc/27538a515036f4705ff9840c924040caf86cc633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164014082/warc/c66a4d35b779d9d82d5629d8b7a9101d2d56d695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164014217/warc/71572bb380a69e29c35572db0510249c56db946a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164014852/warc/fdc55ced187095611b80f59e6c62a74aaf1365f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164014919/warc/6871c709e6a465bc6a17a140dd00906087be3818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164015892/warc/23f8bdfa774b019653b06a6bb430b80b46d58a7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164016462/warc/b5e0c1670c7f2a251d9ef6354b7cf5eaeea117de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164017049/warc/a88d24c55319b349994bdff08dbf2d879521be84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164018116/warc/903edaade497d92934d4e2c9c8cbb7a393dfd9b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164018354/warc/f3b151b11482b9f514bd79849772cf14772f3ed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164018912/warc/9fe65142341cb6bc6f4b3ef2cc318e7f5dc1c86c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164019123/warc/b75ecd6484c05d26b993ce274d9d5ec037116f67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164019268/warc/b6081884a640f5a408a7a5a639db03dae6e8ae54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164019989/warc/3ea271afc4254b3a44981ddd9b426e1cde191ce7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164020217/warc/80734007a22844625f790a192051e23d61443303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164020864/warc/f74e2586664ebb7d5ccaf42c0ff9b890ca5ddeda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164021066/warc/bf9ea6befc476ce155db18e2eea4578eac62190e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164021936/warc/79af491074f4f73f4287898b05d89a33b5c701d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164022163/warc/7af72e734b0c1cef0ec388dffc9a417659f922c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164022208/warc/e3afa2d4a6214ee0174bff56b722a57e5e5a35a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164022328/warc/ab2c52f94601eeba9d300a8c8098302d075834e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164022411/warc/1de9f3ecfae277fe90e203109025162d6c78a9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164022934/warc/45700fab9501b6a5d21a3fe325bea9a237d79fe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164023039/warc/a5949e965bc83ce7a28f9ba47a785d4705c3aa09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164023632/warc/b96ee6ca8d3dd876578828fcc8f3e36833cc7ad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164023947/warc/0f34200e6e21b2696a4c5b83ab2cc16b95e8ea42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164024169/warc/11361031f7882e75c3231ab775f4ff4b57996323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164025146/warc/377344fe5347c949733e0285a585b4f17f357fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164026161/warc/1f8490df5223ffbcf7a36b36530870c7e5fe7ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164026971/warc/493da435e8069f02f61c57cb29d2cd3882fbb6a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164027110/warc/840f07d00a086a7e39f419669e7be34879d578b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164027414/warc/3cd44dd36bacc66e069388fbd3b7eda7b0adb9a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164028001/warc/96b9fddfcf5b195861ae49fa0ea7e53f4573e41a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164029048/warc/3a42d240427df4dbb87cb713c2a9135b7a96f472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164029436/warc/d00010751698ff67caeb5ca67342025f8da4a735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164030159/warc/e1e066788c5d9a654a2916818cfd591995e17647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164030978/warc/af1ea84da19d5f5d1be592af4dd44c20f13285d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164031727/warc/0852e843eba36d64c61463e036e203d1816d37bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164031957/warc/9559e845f21f9deb5faf8ed02e6a2cb8d7247b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164032040/warc/70e49f4ee87c39c81b025ccf12415045a7cbc8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164032243/warc/3ddc342f24f2dcaebc71f10734c884dac87ce9cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164032288/warc/f82bf5716ed2b053404f9f1ee6523b8e3053318b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164032593/warc/fe554f5ce9f553579e3ea5b6bdfc498d39865cbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164033367/warc/c276ce452bee37d566ae3d046dabb55c79e1b812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164033438/warc/a460b1914e95d8f32f167fbb1506d649834b71cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164033639/warc/d85918043483d74763b47bd3209748db42d62095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164033807/warc/968402a91bcc0ab2b654ac58766afecabd1c2731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164033950/warc/b3af64a84aa7352a2f367554dc0a516fa021f85a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164034245/warc/bd5f55b3ef9d33242a009b54a32868bb94522ce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164034375/warc/9dbc68d8a0a4580ed21ad364964b24d868b2becb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164034487/warc/5e261dd2ebaeb752d80f91a754196bfeb21cb2f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164034642/warc/0ac4c50bba215ab05701639756bf37800718c046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164034983/warc/3e9c3cbe1d139730c2fc01e84a993aab1fd06b6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164035309/warc/0551b1e855e6065a2544713e51d9b7c29adb3601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164035500/warc/98a36dbf6269fb3d62d5a240c575871170f3c798.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164036080/warc/8cd4819a8a01a1fd116904f37eb74172c90a0302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164036407/warc/645cdc51fc0f0928c09490ba124a5c86f5fce33e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164036653/warc/ab0a200397523f8c7942af003688231895ed2df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164036943/warc/a4b95c43b8b4c1406a857bac0c49bee4e84af524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164037376/warc/4d88300bcbda3831fbce943b3fc73a5845de756a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164037630/warc/919f3622dfa578a426efe9388f1d410f9002b265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164037762/warc/f467de8d2761ef6274da24f7dc179665ed8b15e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164038376/warc/acad5d54de86d2358e17aaac08eebadc59fb311d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164038538/warc/3980b1db63fc25b8c6fab26979d20a40139fc687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164038825/warc/abaf9dcab469ec0bdf03cb02f35f50451fd1a478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164039245/warc/29098fc415d0bfeddb11ff4c6671f4a8906aed58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164039593/warc/a4c8c9ed9dda005a43bb0e504073d48f175f5c0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164040135/warc/6883f78be7fe914de20c50e1d053f3f69ebce81c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164040899/warc/72755208e1f99e52319611a984856e48c579c335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164041513/warc/b8db3a89e30bb2bf0c2e565f538585f601bd2307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164043130/warc/9885afde2499cb657edfba7ef36e9e01068b388e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164043900/warc/c9b0c1a3b2083292686261e91091c7dd6cad6453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164044626/warc/378e30b916f2cd39128b84be68cc23828266f09c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164044912/warc/4b57f2e0d40592fdea1f31694e38fe7708084527.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164045928/warc/050931f711d619d8cc9ab5c1cca66d70a8aa45ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164046334/warc/5c29dfa701163e197d97bba74174f9fddea0aa50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164047228/warc/0903467605ec35f1b9fa762f129fdf8910949027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164048127/warc/58ba1f143edd07b562ae58755ef03c05730e5875.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164050279/warc/8c2fbd98cc9bb3741c349ca7bce99d5f689a4278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164061354/warc/778d77389f9513470471147c6886165649309ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164069141/warc/98d3ccaa904fad1d3e55eeb4d54e399156e0dca0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164093068/warc/cbaf79d9d3d5c69a5e9ac98c598f2e1032a58d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164099123/warc/afe301daafcd0b87ba96350b1aae41eda145206b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164116508/warc/adc96b2eaf5a17a4bc72c8e3a0375d6a3456c7cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164120234/warc/42288214ca57eff5be22ed570e260c3d104daa4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164128316/warc/b7f66364d056c9283f370104477621b89dbd13e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164330352/warc/a41159baefd2bd118c77c17461745100e46a7d6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164333763/warc/bcc5ef20c3dc1793f7edda8cc3f05925bc9bf999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164346985/warc/eb7c76027d06b92733f4f790686f29bca97504c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164351837/warc/e26b6b40590881a7f1bbb5db24363995a6862133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164359940/warc/c58250823db6987c36aed91c44fdc342703206e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164391000/warc/6d3b1c1afa7573bacdb90baebb32004814d2665e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164447901/warc/622c532e0755c9657d9baf74d790a7c529e1976c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164452243/warc/27eabf2410386f633f040c448d0a30fe40fbaab2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164456039/warc/d2c116848b975c91a005002ef97b9b4f033ecc30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164491055/warc/2cba12018c3a5a6c298a874dbacfb41ccb5fddd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164540183/warc/93a808078fd50b2009ec5553794b2fb7816c85ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164554256/warc/ef3aafa97d5dc0c5aeb7842dd8afa8b5d1128791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164561235/warc/60aaee59c5c5b3b117585749347a18624020f504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164566315/warc/561a7d896be59bd221e40d19c7a2bbdbe1e2724e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164567400/warc/9282780afc74599266644c5d45dadd848d4acb77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164568332/warc/61f60ee89c0122bdbcc35abd7fab47a426988b89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164571932/warc/c7bc3d5f1f4ac0fc070ae1a898fa13dad686383b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164572870/warc/a9903706048832c1c17cf3d083c701952c872ce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164573346/warc/643f6e7e7dcae551b622ce0ebbcd68a9d3acec1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164574548/warc/326ce794570dc566e7066bb7c6d2b5d6587f1245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164575861/warc/c1e353927da9a8a52d3c74483802f66daab7c56d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164579146/warc/00e318bbc4e373b2dd5c6a4c4518eccd89c1fc42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164580231/warc/b3f1dc208892fde4bc42dab24092e5ff75101c28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164580801/warc/57265437432e8dba6855c193c7e09be197f9227a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164580976/warc/99538691f8027d2df58f8e249ed7b5aa70cd2484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164581855/warc/366ab5aabd8247ddd4ff07dbec958dd0ce1465ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164582561/warc/b1dfeabd6ebf2e3b02d4c98b617cfa0cd0cf59a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164583115/warc/e80e7eac58bb009cbcee62cdabacd9713f60a8fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164583265/warc/40906d298cef0033c49affef0416cb596df9c3d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164584555/warc/386a9be8192d77ddb90031fd8fe19a5196b9c030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164587361/warc/188e243a409808430cfa5440812e0c90c9771d41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164607702/warc/e690ebbd41024c4cc75c3729544515c08c901f20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164611566/warc/ab10dad3f014bb6ce4e1aabfe38e4d58819e3330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164641332/warc/f86d8c04370749a77a97f4aec5abdcc266245876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164641982/warc/c6a189554d525fde20e8d93026da1167359277ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164643547/warc/88c2be9ff95bb88b0da3bbc87a0ef95b334d3ff7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164645800/warc/ee545244f520b9699cee17bdb438b74baf03b0de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164646459/warc/152ce6e82719434bcdf060df85f4a0927ac7cdfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164647809/warc/667aaea427a4ceca1c26885c28deaf228ba29a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164653483/warc/b27097d3b0c973bf3d530d65773a6242886bde4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164663335/warc/1e3c1cdb7c341462525aa7b944294bda7de01e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164676172/warc/39db73e9d55b9424bf2e972a35a1d06ee6cc4f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164677368/warc/68ce871b4eff95135d5c0d03ece3b1c90a5a1e4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164692455/warc/1e93c3cbdc8f5a46f93714b74cc1f4c31e62a620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164695251/warc/fa755956e86dc92c1f1700e80de12fd7ed1e7fb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164701395/warc/bd918aafb9ea9378f2e8fae1c02b952abe762709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164705480/warc/918130c0cce9a59d7c0bdcfa464dbbd4176e6a62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164710506/warc/c726ff37e727f880cedb0f1f61d93af3cc1b6158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164722336/warc/ac1bbbd7c49d60cae500da316998590477c3578e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164732064/warc/2614200805f49b22fd6101b8c069599403976b58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164737564/warc/57cc1c667a817217c10404776057acd2d763f697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164740723/warc/7948be92e7d8577ef101622d1f664ab04135d1f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164746201/warc/d308a1e602ed8ab9bf25cbb274c799eefa09b0c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164754111/warc/70fd6561cc38de41050ecd48457e566995bab741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164758033/warc/200314cd32925090176bda0042146051d81e03a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164770786/warc/9c9f37912ca2b67477009fb24089dbe0b494604e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164786099/warc/3c1e7512e5cf99f5ec0f3bfb0a248912e78b37bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164789076/warc/53f3db2dc974436f2774f521ab282eda06ee2515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164796892/warc/787b1143cd5b76a1a293e621e3ae0eb8e3573cbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164805405/warc/94d38ab52b766f2c226b137ce2ff89d5bcfbd8a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164819343/warc/ef3dbc22e53bcb925dbec7e742f543ea85c0c1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164834476/warc/49614256025268516e597359ce349aadb966b4c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164836485/warc/388d760ce11035e0f1cc6793bb3df4bf6f1f85b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164844212/warc/4571f075282df50fd1b44a6c300ab7ab4b711dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164848402/warc/6b83000efd35cb50695729fcd010eb43fc9363f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164858282/warc/7d07260ba7c6ed5678c163d7fb24f169f08c8741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164884560/warc/b1085714ea4410676437d2fce8fec2a8dfcd1111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164886464/warc/e7e5e20f2ac792a044a7c9c195fa9cd1912e6c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164888618/warc/4385e5afbc63401a96623b675a02e847360dea3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164896464/warc/2682577e1b7d8311a6bbc0df528413713f48850d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164903523/warc/0e03151ab06e02d537c7de7ca2a979cb2733c4b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164908494/warc/dbaa13aac18b45d0095a14e228e8a0b35b34db14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164911644/warc/864a12702a9b4f9ecb1d8145da40aff67e1a905d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164919525/warc/dbe2b4e7dabb801bb11fcc1f1a6a9ec410487d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164920374/warc/576f674b7c9c4b379696696809d50ad9adcf16fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164920565/warc/ab9baadd431dab024bb0a0916a784bc301daa591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164921422/warc/00b59ed534954f83e04943c6f61f4d3cb9a6a247.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164923389/warc/d8752dbd898c6ba63e00179b71a77c3d90867320.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164926426/warc/2eddf980e73c399324425a068cdd362a99760730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164928424/warc/551388ae1665e938c8ad3a038cae02dcbb1dce4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164929439/warc/2f75ec21bd7b2933f6936ff087702d34321587c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164931675/warc/42fdc7a216a526bb539edadb32e105d92703ebb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164936474/warc/a3412e231ddc8599aec3aa291e6f5d95203c14cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164937476/warc/cf8620e0d27a66a1563d920a6e0ce27b8f7b27c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164938822/warc/74a72bf84d9157410cda0677b9f68c901f136644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164941522/warc/da5e360cf269ab7404814390b26f87e37e8bcc69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164941653/warc/1482398bf4fd8fc0842a37abc67264e091bb6372.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164943590/warc/4ccab1dab8d3931aef6ec88bc344152d9b290cc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164944725/warc/62bc86b4c04577be9162e1e10fc968f231d02f71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164946521/warc/c4580e0806344fa9a2f03f528fe063ba92f5ed07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164949664/warc/e4446a9abc2f18ac56fbfae60a5aa5682f94f2dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164950517/warc/8386b2c6df3c3101c4064a68ff56591ef162bfc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164954485/warc/0f0047a221cbf17b5a5d6b3c66bdfbb6d21d7531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164959491/warc/b82f63968ea5686a2f15d8e07568212c331039e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164960531/warc/f23c730ef4bd9666aa0c9e2a6c1238630d1d0588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164961715/warc/ae7f7cc05e3bc6839e735232fda3361a6096c37c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164964633/warc/09828658044e6e6846292c4c33a1148c6d6eae02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164965557/warc/58e1b4f06b74c993b54e23ee37e90b94ce18b17d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164972407/warc/1abc808b14a5b3db69d8fd493e8eb0c0b9ef05ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164974477/warc/a69d25e3dfd9ccbd9a35cbac49f21c137f75369c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164977792/warc/6383544fee083ea446be2135694101ed1c566bd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164981569/warc/a176a76be400e5a5290cc77e06e8463806d62309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164987957/warc/a0796304f70751c2a1bea2b6082905a822300018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164989606/warc/8702686a975a8562c386ea858c20b134010d74ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164989714/warc/ff9114024691d90fccff9b1aa3caf195cc0e88b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164992771/warc/cfd7833e3cf2bdb6cc5a9e956c5cbc317acae4b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164997874/warc/226fb6de1cbf3be4d7742d2136fc64bea5f5809d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164999586/warc/9d9c119f3ee434d453550910a0226340cba8375f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386165000886/warc/67b43d81752384b1b34fd8742e0492add11d93c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386165002851/warc/7322e67489b0460660f16609b523aac3a81690fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386165158218/warc/3b245ed45b7d2c236424693866d9dbb6b9c54463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345758214/warc/56d909ab9714ca8a33291b1b4b466e54fe255d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345758389/warc/205d6b473fed7ff49c5cb6ef58d8adfae51a25cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345758566/warc/b3dbc25f72a1809eea34df88f83959076fb71ff2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345758904/warc/9ef1d95cd73dff188c0870c4ce9698c3adc5b2a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345759258/warc/93528bac3aa8afe70207244be6ae6a44fd5ef4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345759442/warc/e96c48c9dc1b6cb41059d0dee58750c12c9731a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345760007/warc/a4169ac6f9deb60a1aa113f3be3c94d4197d1977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345760572/warc/67541c2baf4a26ccd768e1c7dcb731f54a266b5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345760669/warc/67d2156a2271a4f0edbd2ee662dc1f5f19f9d342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345761603/warc/5647453755eec550bc3717ad115b467c1e8ae189.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345761938/warc/250e0f5e34656876d47fb91c10d4665044f2170f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345762220/warc/80ea4b84268f3cd632efa45060893ad7688f7492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345762590/warc/7b12aacbce337aca30988dad1ab94217d2bdea49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345762908/warc/4881f350aa27b9b0b1a5c73193153693f79040b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345763533/warc/6a7bb94bb04ccbdd2ff924ecaa2ae15e1eccb704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345764241/warc/acf40c03788d27f0373e5725cc9120fcdb7e8d5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345765796/warc/8c3fa4c89f17f5358b79a2c9487e51105092cf22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345766127/warc/bacf36f6ec544a64944e5c0b19addf300604bede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345767540/warc/3fc96c57f3ac45dfa006185f149790976c4ef334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345768537/warc/1e12ea276d110aa992542eed52936124ddbf0ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345768632/warc/696f56a05e7f9b9795c0603f3c86cc44aacbd3db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345768787/warc/c5e7348ff12b11cc73acd8dc5d106716566cbaf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345768998/warc/46b3d9c683d7bf7738e96ff3649b5210d46469d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345769117/warc/29271e125f0f9c2316108a2d620f35814ff2965f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345769121/warc/02e5a6488c85c103de60f4fd828bb1e837aea84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345771373/warc/954fcef6e424c719b4667fce008c04725f89fff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345771702/warc/f49178dd4c51aa0433d38d874dbf32d4627f96e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345771844/warc/8607a2c0eac27b640e0d4e2c66853144b3df31de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345772708/warc/61a4c14e106b5ca0399c044a01dd871b21c0c43d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345772826/warc/a19cfc07e9835297570c8cd4d1a41ff2dd1c4106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345773090/warc/4b2a150272a6fdedef7c9d7f6d49ceda25cf2bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345773230/warc/f9fc09709de7bcf670e67fe6d125ff87ce540dd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345774311/warc/e49d52007e00f318e3dd4a7faf13416fdc994ab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345774525/warc/b4435ef283e971d1f6a2954908399df1b3a400a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345774929/warc/c437918e4e734711431975f204fbe582f709314e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345775028/warc/7bcab649e4d7a62022ea1c0a8f4ee884e4d7def2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345775355/warc/15e12145da59b0d739cc51551c88c9a8008d4460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345775423/warc/db21aaf86f340093bf0b49e133e307419625bf40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345775580/warc/f4319131b676c11b28d1d334cb698122ebf13025.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345775611/warc/b06104b00ecbbefa3eba82d9303a4f5f98539fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345776257/warc/8624f7f657d997fe97716758e9fab09ae1a27c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345776439/warc/931ae955826876f080284bed8e110c9fb8d07df8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345776444/warc/45697a5a4a42d4e69a6d28ae6df0c2f0dc2d2f27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345776447/warc/b5bdba49cb1e1040eecd98beb3668cc44daf5574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345776833/warc/0f761e681d9572f88c38040d5cd1cbef62b8eb91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345777159/warc/083e02889a18ae60726edf4695a77dac1c4e5ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345777160/warc/df9956f452c0fa299eccd573b8dd006939c31f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345777253/warc/03d24882a28b063f0102683216817ba70f561a36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345777538/warc/a81ede82a6f6dc13ee5a32268612f93c11c18e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387346051826/warc/643d4601fbb2169c6b1bdaa9c8f44f9a74d172f1.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2013-48/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2013-48/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2013-48/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2014-10.yaml b/configs/cccc/dedupe-url/CC-MAIN-2014-10.yaml new file mode 100644 index 00000000..10132f32 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2014-10.yaml @@ -0,0 +1,578 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999635677/warc/0d33a11130f30dff6e0102fc07fb926381c7134a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999635916/warc/5b83d579c1de3c415928a2f11b18f8e8a3dad89a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636018/warc/340f2f26c368e22af45477a9bd7a619aa2559c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636222/warc/df319ad9241c04deb552db42f6cf4448326309b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636575/warc/15fad4cfa9e45d0ae009f4780a1549e45b3083ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636668/warc/144f66d8f0e4f082f39d558da68eb12e9e465ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636779/warc/6622f831f99820bfa0aa798307e0d048d850c2a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636902/warc/f5c2369c4e62b99d7612ae6d018c641bb6a09606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999638008/warc/6f95e837a9390799315fde9a7a1cf7d17909981a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999638988/warc/f9cb22cea0382641461b30aaf8d340a92118d58d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999639602/warc/7fca33879f6cba8befe8bb9fb320f9af1650f385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999639954/warc/ce448f8bb2444037362dc8b536388c378b83c5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999640501/warc/222e4cfbccec8dac2688024e81fbd1a4adc7f77c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999640676/warc/a09206d4a80adab987d7370a16050493d4d34cf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999641260/warc/1f0b7b333ab949e7914c0fd3a62ad75dc48f605e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642134/warc/c0fa9bd94a2110fd0c4391f21fb71c1d88564a10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642168/warc/7d39fdccb5195fc8d17791565c7d1e576301a112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642170/warc/f046d6de6df6d92aa15ffb7261fb1ac92fc63c90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642201/warc/2d933ee8cb6439bad08a4a4897479d8e8f9cd29e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642306/warc/a4fe8550c353074a03fbe3847ed8a818413d0fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642307/warc/5bd0ea42c794f51faa7fa2e2d907321d31e2c215.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642517/warc/1a0c5665a190656669db248e7b6cfd5253da9b59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642518/warc/405bd5f2b9e9eb2feea3499ab4fdb8973c65c925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642519/warc/5c442c76d8b7b07bdc97eea6663adcfba0c2378c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642523/warc/816eb5e32115e1c33f168f2a47175d47cd371b25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642530/warc/09b1f8257e23c144a628f3460a5e93728f0ca272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999643993/warc/707163951ede1f393a11264c4c5a497069f3299a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999644032/warc/1f68bf0c9ad435fcd216dcbb5bfea57e6be6cc99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999644062/warc/36baa4dd1f23291ba6623bbf37ba48fbe4172eb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645327/warc/66fa5fa50496ff11f16246d53ccf447e83da94a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645330/warc/e1d3a9e2c0f4a1e4f9258388a8839d0ba96cb591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645422/warc/1d37a93904c38f390cd0d676a1bbe63f912f1013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645491/warc/26e658b0db2e2c0499f565dcc0aa39584b2e560f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645498/warc/363ab068b54e91267b1260899b6a9624e9823fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645570/warc/fc94c7a0d3863683690ad5f96735734f02bb80f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999646172/warc/bd507d08da18994b8355501d9a5a51bdea288aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999647312/warc/fca5c8e9a89013f1ae7da3e4cd8edc735b573973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999649814/warc/6a7dfb88ed8800f70a9ea24b9a793b3f69d7c06f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650252/warc/01446b31fc6f5e1cfc3148f5c8f423365092f4d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650254/warc/dceee0d93f80270a72924709124f8854c678b162.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650424/warc/c30ae2dcf9c405f616d04abdb21e1fe8c94c60ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650477/warc/ed12067adbb693936b049a03d9ed8a0cf3d57db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650773/warc/7df0afad2a421e9deb82facaf56c7f127410923f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650775/warc/b32dd288f962651c1ef53dc3c7f1b28f1ab7d6a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650794/warc/e84d25b89c49594293ef93ac0233868d2c36655f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650844/warc/d8d8e8c56c9dbfceae65fa06b1b079b8ecdb4113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650916/warc/ec5f6b2fd3f199b83fae1f512ed0b1d13c764b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651148/warc/20c02ff8ac46700b33fe576780bfb6e3d23ca5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651159/warc/fbcfd67745e14add1799468d266058a57371a674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651166/warc/53200429ca2f375407e9644055164f8dad296467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651529/warc/b544fc22fc09816a6e050bdc67fbb7f7e288e9d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651577/warc/0903f4cac46941a86fbc9f8367fbf54a70b559fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651631/warc/c9905647f5c80ff91d0e15375da4a4d9b3212d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651825/warc/30e103486a74c945ae45a8f67baf708c92a27faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651896/warc/dd88f1cd285714de1bb1538745b115a1fdf0754b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651905/warc/41a3152384b6927f42f69c615a725d5c1ecc7a0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651907/warc/467c7ac52eaa797b81467e43ffd8bc10b1d0ef53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651908/warc/02645a35c923a5759b7676410e63d6694d63badd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651909/warc/d2378add203a3ce6a6de7a12d8782f77cefdfadd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651919/warc/5d39ab5ac9431f13d620a7cf0caed2ba264f0bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652570/warc/606ef38da7c2ae7755763b3f510051769b31d1ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652586/warc/da58ba4e8708cc606fd3566404ef6fc606997083.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652621/warc/f1280232e501f3fb54f9f1d5124aed30b34dda48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652862/warc/bc01db3e938b72b42ca96b7f3730b40074d9be66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652865/warc/61a2460e438a361e044daf6b5d5dd100f773fa73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652873/warc/69e8ac1fbcdf3c9aa0993a6b1db7f820fa28053b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652921/warc/ed73665ec862a7f13dfa4830f8665938a823e6ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652934/warc/a9c0147797657bea3cdec3ff32ed69c6652c4aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652955/warc/323769f261f990c25fd8f37971af5089fb4525f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653077/warc/4472b7259e136460d9494b64b7a138d66ded3cb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653106/warc/6963a85da62b447056e3d6523cdc15a07cf9721b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653202/warc/807a501bc75b522cf24a7dcdf411996d3a4d6cd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653325/warc/96cb37519186f999799bb9b9661cec52d0fec1c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653402/warc/5c8fdd23074bcb1fad23f97b35bd1a635dec9c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653416/warc/eb5cc92e102313999ae4d6644e10e732ab052531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653644/warc/c032f24cf7ccd07aae3db50707cd8040a4d09269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653645/warc/8323ea1d1bdd174e495f4e2fc727148638436c60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653669/warc/f0b49080d5dc5167562f7df5014eee3390913a17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653833/warc/6f05820c3e812264e46805966f9192de14fe7350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653835/warc/74ced6d84de77caed7460b2cd54e34f6aabbd9c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653836/warc/c736914f4156920c7c437b9c5a1148aea72d65d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653980/warc/f65dafe1d4ae36530a67276ecc8e11f7978e5bb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654003/warc/c314ab33a0a5f055cb7de227dae32df9d938162d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654052/warc/3717db1856b9e69d2b438675cbcd792da0a04f99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654272/warc/2e78c17db5190556f377ed2b5f156bfa64f6a11d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654282/warc/75c8562d55ea2484879839b3f8aab305ccd7e72a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654285/warc/0a0126c2c0aa36b5fc86c8bacc290b42bcdb6b74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654293/warc/472245e6a285e33aee3e7deaf44ad1dd1fe4ea74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654302/warc/62a535e968e263f4e3b63754b86829c61aa3cb25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654315/warc/c90ef075059feafc9e8338d43a10f26d99db4864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654330/warc/d6f6e1b395a08050bc2db322f78d05477808ef6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654345/warc/193b0c6decf8ca48f9301104af204aa94d198068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654390/warc/c2c90176625dc761957190dc7c8c1720289a6e65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654396/warc/20fe0fafea7532272f40387a5cba80d789601a99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654438/warc/9006fc703af05a5b06b7bc952594e2256811bdb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654440/warc/84588efbea012682fa3beeba1055f4b40bba163c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654450/warc/bde6b64593a62392edf09304e49be9a655bc45ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654453/warc/4097efe19838475f2389663ede5bfdc97db782d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654610/warc/2c9ebe2b87847fa7e8f496210bd1558fdedc89c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654613/warc/e81845a203283fa6dd40b9bdb333973da21c7e9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654667/warc/903978307e990a9ea03cf8560d7d4fd1ee882b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654758/warc/e15738ff497ac11c1f227356d50a101acd8981a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654759/warc/a3350040fccab3e2119a99eada66d1ebdd68fe45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654815/warc/ebe342b604fd44a49d27020ecab23db5301f6125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654872/warc/cb80f5de2c705a60da82bdc50b032f92659b0543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654886/warc/d4f3333900af594d73976d2fe360ec4702f6f1b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655040/warc/757853b1f9aaeae8be13ed4f999dd3a98ba3d03f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655041/warc/1bf4154585deff612bba009f3817df04aac0d82b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655109/warc/1e2336b553944ef8af50a30d5ea032e2dfabac9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655160/warc/1785dd799fb36bc990a8e38b9c658b7bdd115eb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655215/warc/dea724afedb98b74200d855b874d15c515e86572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655239/warc/2a017a4e991a98655b0bfb1ef4618a02667680ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999656144/warc/bebcd194160bc89d402ce4535b1630bfb97c7a80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999657009/warc/6a01b98dee910fc06d8020b34e234b954cfb5490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999657010/warc/08369495228dcafdaae24cf22d4383577bd90873.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999657012/warc/e6bd506b83a6cf2fff5cac21e44fb3c54fec6e43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999657340/warc/cdda540ac37fd07c2bc574bc170c8fea15e32f49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999659065/warc/b7a9bca877e25a46431e0fa191050959d11fd23e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999661726/warc/b76802748c5de85ce4ef23676d04b171ca820b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999662156/warc/5028dc78d274a1be86890d33e57c1d9a7d7234ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999662979/warc/bb338ec24f6a8d054d97c961543c0f8380775b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999662994/warc/cfb47f4a8f16eac9b822e8e3282ab714f56de341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999663286/warc/789f656c607ec6d1238c742872320321a075c14c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999664114/warc/fff66c34ace8fb024943d91836c406d3b92e670f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999664120/warc/749b39b9e8b2157fa4d6cb9a7e14209d0294affc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999664205/warc/e7b36431657d9231fe7ec39082a80beaf2a9f85f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999664754/warc/ba1be5ab6b3b38819619d2ab09c1fa154647fdf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999665469/warc/2135d8f560d0b53a9942a9e44c2ca45411135c83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999665814/warc/5c672475079914439af1f763631c21a59e324615.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999665917/warc/d1089c6f7c55833594edb50b0ec123ac5c844411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999666921/warc/231a1da13105cc945c1a96b7408d681b20b27dbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999668190/warc/08fcf4b7ecedf020d9de2a8c62c9dad3e9e4be91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999668222/warc/a7a7f9d3ce9f9fa49cbf56655e4685f4d70b5f67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999668224/warc/3c1a52b6845b607361a5f6e8b2393c5dde671ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999668738/warc/53586a40ed7e05729ec930a0e37cb084bd587429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999668865/warc/d31710483629e084cd2adc4310f183b5b1ee6abc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999669324/warc/859906b249a88d00301e6f71e6e2b4f0dcd6b567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999669442/warc/df16d7679f89ca8df3daa3f0eb7d37fe40d36df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999669780/warc/9dd13a3003912700e812ffa4fbe49cc3cd1f9e67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670048/warc/16115b25f0b1eea88bc527ee807ebd0059ed5585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670363/warc/ce27031e90112863d76e7bf50ee27a9be1e9ec73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670669/warc/617a9beab1c05c7612e4bf2f095a8af4ba51fb68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670740/warc/de70ad77081d6ea1640fd69c5250eeccb1d10ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670852/warc/e89af52252da6ac488d19702ca033fb3dae851de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670924/warc/33b62df18f53089896a943f4bfa6b80be4336b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999671301/warc/221aceef063f8125b181f3af89be4a085f808496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999671474/warc/f8c6514a733925883f80a141be3ae9395b9d560a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999671521/warc/76ed90bb667b1aad54cc20362a866fd0e907393c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999671637/warc/0ed7dfecb882858e6efbfcd33370044081c1f22e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999672215/warc/b26d649b5fe6b27641a8a05744ca14a41de1ac51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999673133/warc/3518488293407a292c151c802b737053d403887f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999673147/warc/66b277bad6b44730de69229afc9f17d458a02e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999673298/warc/b6f07c1d08f7a0f653c8ed1c94b03f4d35be45c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999673608/warc/377a5113d0ded6714eebb00e5a759dbe852e88a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999674031/warc/54d91af39717a113df7dae3402cc9a53105834cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999674095/warc/d707be4a40f903e403a717ca4936b854797e9596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999674642/warc/ca427f9c56c173b0730af0cd0d8b7c4afbf9adb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999674993/warc/17c3f59506b771367c4167a2f7f187504fd2b1e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675037/warc/c8931e099febaba32537bbad327f6e94dbaf6510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675300/warc/5a2ef3ca9143054513000efe885964b03a317be4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675557/warc/b946bf58246d99eece2912f34bdfd53284e543aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675662/warc/84fd716175a081de8b1ae047f5321384054d3b1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675839/warc/711469f62589d87163ad1ee6c029c052c052e81c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675924/warc/4e004d7ecb9364cb97628301dafcf64489c14242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675992/warc/0e292f82b448aa8b6feb31495d620b6387f99cd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999676149/warc/4f22583734a3575726dd8e5368ab0b66de713e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999676283/warc/5f988147e95ed2454a7f7d61269463b110a71acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999676768/warc/b1c30192ec2db35eeae603a88d318a609acbb26c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999676834/warc/a295521ee09e9908c4d8934557ee6434ced67196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677208/warc/ad36076b91db6b4ec258b57a36acb3f7d9bf0ac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677213/warc/e50611cf4937fab1e49155da324ec52ecdae24e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677352/warc/ea779fccf4ef8ebe40f7b2eb5fdb75ed43cbe929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677441/warc/0a266325f3d878dddc42523e9197d6bbf9e2aa3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677501/warc/9529bd0e84584148d376b711024b6324e1ef36e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677515/warc/9a33cb40ad2f2b0eb8026da860b5f905e38fbf6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677605/warc/fefe03c9f5435464ed0c7b37bb4e45ae6bd17988.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677941/warc/b9611d1a5d7722a2f9fd455e234373f404c42c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678302/warc/d540d02ec3c1761e1e236022f59817e22804f32f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678381/warc/8dfc092ff0ee10a83205ce4397cade8dd2aee740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678556/warc/20ac2ad6b5377451c201f69b871a68b2e43b50dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678747/warc/65b04ca29cc7ae7b2e9618f6c3ba28292e1d0536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678772/warc/71e60ab9f4f4521677737364e1e6af823e6a8c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678977/warc/f49c94ac7e39c75e72b3be7b1ed0103c9819d0d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999679121/warc/34dbe8b87b6501b4a0ae8e83f50e829a8f567972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999679204/warc/e528b448896d158565a0a176af44ca85895a0e77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999679206/warc/7d8ade1ad8c81f9e393b3af1033507cb5ff0c1fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999679238/warc/a217a4a0a38b06ee400f57a4df8bf9b6acb5846f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999679512/warc/06cc148e5efcf1a741cafd983cfee5136388204f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009669807/warc/62530f0c83013154c003a67c68878ec50341a5ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009777085/warc/87d35026bdf3b5dd6724874d069825ac7aa36535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009829804/warc/6d57eab5b24da29818fff56f8d903ff074035409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009871907/warc/13457712a29b83093232087d546bc1813b355b0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009885896/warc/8b8f7dc78c92a9cf44851aa9426bd3b597d10615.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009885941/warc/8ff8877ea494c268402cda19b03f9c254e292d5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009903935/warc/a83739f0e680cb984d540b67312cc5c6ac9a6b9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009968414/warc/255a6aa99b9aaf207c92297aa77be9c92a5ec214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010048333/warc/13c40237d2238831e695cbc11939ded3f3156159.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010076008/warc/0c8f6e2656009f9caae7d0ac15b2d9e060a01294.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010115284/warc/fa8c6abfa13e5cb947f07d67f4ad874c03998eeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010128060/warc/7952a73e7108c329d7d0f3f4518f011cdb620121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010213406/warc/ef3d70c81ccd6fe342255acde5f732596df4f6a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010216492/warc/3b61e7872f8de177e7556dd316a838dc0a4bbdad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010292910/warc/48e4b21593f814d264010422dd9f49ca5159f224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010295336/warc/1e403b1b6457862d24fad216d44835a27d4c149a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010303377/warc/7041c2df4f6919c744e4ddd8ebb63fa24aa7cb9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010305626/warc/ba32aa58644e5b4ab424f7ec3434539dc9782163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010342638/warc/c2b306c4f85fd6ccc88c3464814d3f8b4e8a9073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010352519/warc/d79158dc0673b53a2086f771e4a26d5e51314ff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010354479/warc/6be375c9dba2928296bb2707d2a58c4da8965826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010355709/warc/b97b180b97887937b1036c0df4dbcd71636a24e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010359804/warc/4f6d89026495a507176659fd7c6790a7c8ab4fde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010437227/warc/da91d4c8d7eaa2079e3cf1b8d7d9f32ca02d52d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010444750/warc/2a4014cefe5658bd146c05d255b946993e942891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010450813/warc/158aa66bec0840041eae8edfdcd03aab42dbc50c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010451932/warc/3d86e92d79e38f5c3fa1c0cdaf860c3223088678.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010484313/warc/e88b9cc1ee239b270f32b73a6c093664e77f1964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010491371/warc/0a3c5179a90bce02f6711f34c66bb5d52a8c6593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010492020/warc/c1d144c52d40d360ef38fa593cb9b657e978d3bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010500848/warc/d01fbf3ad6682f3f97b22a51e591bd0725069452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010502819/warc/44cef07787472460e2dce39dbfd7053e0768262e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010509865/warc/347f8dd2261c883df38ef43f746a620a1dadb0cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010513091/warc/f24b3544d2ec852c0928536d09d3616029f4890b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010514920/warc/a3bcd1fdb565495d6f1a0ff4843e0cedbb68168e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010527022/warc/b3bf901a178cfb227a2e4f78264e392df2f65f56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010547566/warc/f045920364afdafecd15445f19afe554e44bad01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010549973/warc/cefb13a9d3813ab3f08f2e087625a57bdb647221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010554119/warc/bf49dbd411bae56506838147492f8e0cb4ec0793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010557037/warc/105ac6859851ccb8903584c6c71d77376aa1d638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010557169/warc/407682b6087d1b299d03e0b2ba6d0a944fb3a5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010564986/warc/29f43c2f4cc551615a20d231e9b634026c71abaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010567051/warc/e1ee8c97910510196218d41735e1afd32d631f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010607072/warc/3cb519cc975111b2b666b155dca7541eeb9c685c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010623118/warc/f9e57695d4cdbec474c9b93c39811d67dcb5c1db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010628283/warc/e6ce4aa6d3db639a70f516f73e322966907b7a68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010638293/warc/6a452566ca64ed66d4445f9651dc995537f6d171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010642209/warc/d2823b69cf0e730f7857f0f5ad8502bfee9be7e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010650250/warc/05d7ca4e466581b4a853b6316ee37d9a46cfc4b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010653177/warc/bcc6bd3080192ef8368be8a7c7a6ef2ff567d68b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010657774/warc/da078eecf1c989bc9eca76620a97e314691d4e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010666178/warc/0e42733ef8fdefde67ee9c3774dbf633f20d51b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010672371/warc/025ddb8f104722815517416305ebfdf85557a457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010674226/warc/21218e9c116d5eb996141ca2a02d2db4fcaed762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010683198/warc/2d52c5f6237e32868c5128ebb797986d85f06bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010683244/warc/e0543c76e3fd1e68e32b7271bb6bf419e7ca06ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010693428/warc/02fb9ef218f09fd5b81ed773994d9e84d3b13457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010695334/warc/95f4ada5692221fe4ba6383a6f48a5d342a4dc1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010701848/warc/73290b69ab8bd01ea8f93c1ba76d0cd2bbebd1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010706290/warc/f9970b8a808d51dfd8de003bdc03c5c251e0850e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010707300/warc/b606467e66c14987e4062d8a7e3739357e4d1c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010721184/warc/c8b25d7b7f904005e87507bc8fb482084c90a696.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010722348/warc/ddd918d085534d01f660c3361adfba91f49826d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010732251/warc/d6ec1a951fe17363938399de30710995eccb67fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010740404/warc/f001580b6a0985d90384616bc284e27ef41d5c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010742343/warc/6a28c5dbfb12061f10867af6be349839d8f231f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010745689/warc/508ae8a33d9a756da06b66bc151a44a9ca4000a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010746376/warc/7bad11b23e3cab3360bb6cedc49f408f844ce0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010749774/warc/de6310f1f0a076515ab53d806fb53c751879403e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010758552/warc/469053adcd2e1ae939ddd2e60c1facf6f6d7f7fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010765401/warc/78e3f8d9815b83fd0c570d8bdba2f21d8205bb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010776091/warc/215748c6c8184b0964ea453e999f814846a5c79f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010776308/warc/a8a012207ca3da7e3324bacde9cce41d4189bcfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010779425/warc/bad79c906072933ab25f014b80f056b9a43ec658.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010795590/warc/610b93f904c136d7214a3b8ed99eab7b89382afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010803689/warc/565b0a9d6e9721267a8dfec24a09a045181d185d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010815495/warc/dd6ca35d98819e500fab245a27f8127140577bbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010824518/warc/fd7571d884c5ba3cdcdb5880cd00a11174f3531d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010824553/warc/16b3eec341303c3486c2c04d4710104a2b83bd3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010832640/warc/384f9663f2ad14ada40a4fd8ae00f74c1596a622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010840808/warc/694f3afe6ec3c2ebe5ebb70124e58578d56f5a0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010845496/warc/7bc96c1bd8047b756f15566b68c135118d2e0c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010851505/warc/a3cf20bb8c6066461a5801d2317673604ef28f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010855566/warc/677d31fb35013d47f73cc3a52965d2b0f2bbd7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010856682/warc/2448d6ab73603352dc5bd3912e9127bdc4142e5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010869716/warc/4a87dab9d45958f9e702ec285f9d9c8eb2244b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010883242/warc/316bb4ed18e5563c5fe9d75dfa590772d86a7e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010893879/warc/7d4c5f65c751ad278c97d8af1549abf221053755.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010901252/warc/885fdb7b3a64b642550ba8e4c0fdd7c050cf8c5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010907746/warc/c51b586e27065dcc855cd27cad0d3050ce8de04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010914773/warc/ae1a396cac59edc4a3c8ffe8d2f570d3c77633ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010916587/warc/354805f05e2bf292ecebb37c9566cdb449925b9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010925635/warc/40bcad632bcca1e76feb3645ecf815ea51469204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010929218/warc/ec4e4842b8e64683b901f1e5bb19310060dcab2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010934950/warc/d1f229ee56258b4fd7717efc54ba8f6d3bbbd85a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010951612/warc/f33da3b9d768e9046f9b5f84ff0bed419b084dcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010956215/warc/4756fe6373aebae978709eb263661bfd5289135a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010962725/warc/a6754dab6524f8db3eafaa714df94e96f6a6f2d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010980041/warc/fa88fdcf59a16f091d32df87191df31261b08035.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010990749/warc/9fee0d9f1abc3e24cd48a7b2a68d79c70e3ee655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010995802/warc/012f57462db425572514cc52f4608e76ac6bc7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011005264/warc/08a206ef393e67e65dea28360674ad97a2b0ba7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011011190/warc/6bbe87fc91d1687621a714a651974d2c07497ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011017001/warc/b566416040d577ee81ae0eb3cf767d43c4640ccd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011020120/warc/81aa8952a67984ed596df7b0878335d26f55aa84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011022670/warc/a33671e98ca50eb955657e728d56b4ffc1ae27ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011025510/warc/00a40392626ebf4d4e53608015b07cca3d26525b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011025965/warc/3d778746e351975e0766ceaf3e9f05a8a0205651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011030107/warc/2323fba942211fab32cfd3a65b33d92903639570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011038777/warc/3667fd54996c0971ff37fa5eebfd5dc274e5885a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011041911/warc/c48d0711626a079877a529e903593540880dab21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011042531/warc/5ae609fad72e0b85613fa942ea21697f52c81079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011044030/warc/0a9832aeb28f2fead9604e20c4bf6e8d5f9d9a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011056097/warc/faa9ad35f1c4b9d17eb781585252ffdae35f67c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011062835/warc/e08b45828aa4edbc39064e83645e935c5fd65f6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011064849/warc/5c2481bce26b4245208460c8d356be5c140d954e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011070356/warc/f7650132b10a15f14561cb97d9d341ca0f40cfe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011074272/warc/2143609b66b65526801962f276707fce44cd0ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011076681/warc/f2f72fbfb1770af9ab0e661203b657cf32955b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011082123/warc/eb339188c32b491504d5cfc905ae64f1d151f4e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011085177/warc/91af2f2d9d4aa9d89bde1b23eea523a6c7d7d0bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011090254/warc/cc96a2d038d911ce705c6a75010db079a2d0a053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011094911/warc/d9701ed61e078da87c4e401f0421815950c130f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011098060/warc/3cd04d05165b9449d068bfc2cf4285e83e2ea8f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011101613/warc/1c82a83ed4e95354c6e831b86b1bc92605fdec6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011103144/warc/3896aefde4e23e154de4eb3468135845a879fe87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011112269/warc/1918e777f5620dbfa9984ce9ecc6eceef08da245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011117323/warc/fc59e87812e8b8f95c9afdcb4ce5e724db233722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011118294/warc/6494f5e7d204a6ac1749065a55024c86405179bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011123461/warc/9bf68e2516ea863f0eaca085031503213218711a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011126320/warc/4738f8a8d5238b9dfe96a6a860e1c81b2485c3cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011126350/warc/e52d439a6c50cc21386434024a379ee46e955d4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011129529/warc/17d4431569a136fd76307234db7756cf087b9a1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011131391/warc/80769351b4a624d07a51609f93fd8b6f6fc3c2be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011134261/warc/071024b5d3dc3432bf641bbbf64fbcf2e7994241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011138462/warc/09883c0e3356dbe38ea79e3123ee96b5a497a4fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011139063/warc/a48d6897d17432d3ee3f2c0985dfe7512a283823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011143926/warc/2054227fecd2be75b959cc86032de89189616939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011149514/warc/61767cebebb14119bc8f9c4ba1f003f3dec81b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011150121/warc/0364ca8ec9161be47bb829de5868c9e7a2ce6707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011151170/warc/f2685eb73857cd5909f6297425505c8cccc4ca4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011155638/warc/429f926ee555e255a8f5eb5ac24a2285d585d40f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011155657/warc/77444eeda72f98852382e3a9447bb985059721ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011159105/warc/54ab67f24fc4d813468800c02810aa2631d96a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011160422/warc/27faea95ed6049340153a0cdce69e813a11b6274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011161070/warc/a1d27d6d47a33e39de0b4307d2005a2c97d9ce28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011162707/warc/85c5464bf454e7a8041e949e5f5f1cbabb345e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011163856/warc/5cf97530270856bc2f357a0fa81674ed8255294f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011167968/warc/2c9176f12dc6b9a146bc00b8a43f7630d7509d09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011168572/warc/ef647ee95da26ed008769d586bb1ed57aa3c2811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011173115/warc/f9888e0c082c6e86358e2e13b5559bc606b42b1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011174089/warc/5d1fee9d2920cab3c1cdcb01fa3502c14847a6b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011176878/warc/b66739426ef2035c3f4812605f8e6c329dab132b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011179079/warc/51173d89ce143fa584397d8274a3a267346b02e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011183468/warc/8fd3ee0074911b82d6a8d335cf4aa50c31c0913a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011184056/warc/7b6721eb2d943de422252175a3ce4b80ceea04a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011188282/warc/8490eab81219056d192ec31bae32b57a1a80fbe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011190529/warc/6f7e25bd9555023803aa9c70501217bba9c2e062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011192582/warc/8bcee8fe8ad422b17448cbb11ca2e3f31ef550a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011198370/warc/3d7aa37a53faafd472634fbf4be48cdafb273977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011198589/warc/7fce9eef627ee2d21c8693c8cddae2dea40af8c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011202213/warc/b573df8abadfed2bd13e883fb942918e11f03628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011205602/warc/5b6459a4e0e8e950ddca322edd9730a5cd8ac4c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011207526/warc/0db7bd5334ea7b5f0a904cd24680cd5c3a72a897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011208420/warc/a061e5cc2b3d8688e9d68a9442bef3d2781134d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011210359/warc/1388b0e94834391507704b23e445da455618e8a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011213270/warc/d3a1382e041acd6c1b7275bf99f694bcc2bc069b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011215089/warc/30aa6f17f7ff3feb7cba2e4fb3dcd34599c36215.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011217448/warc/e30290d6c517cadc71694fbd690ea45c8821035e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011219293/warc/31f9b45ef9bce8a1f80d6769e4e3e25d1bfba914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011220528/warc/7d7605ad61c05a6157b02fc5ec39357573d75209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011221943/warc/ba6bf5f9711894644e188a01c0058800cbf48482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011231453/warc/766fe279717d09d93a0b77cb1c56fabd0d55ff68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011232483/warc/98398795ae2cafb2a2ec37b508aec61e895e8931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011237144/warc/714fbc8a1136700a10ec0b29d7aca40cf17c1114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011237821/warc/a58b4cbca7f331291158116f5549afe7b77c0768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011238390/warc/f297aa4088d54887d6c39e640db10386718ec1e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011239452/warc/943a4f933fc91dbe0f94d6e2dd868619b1877d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011240122/warc/6200826ffa7d2a7053c779622c3421af2004d622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011240269/warc/75b54d4c0b38352a1c1603804c0ffa03495f2c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011240315/warc/0a77c001ef65af28b433de588c935e72fa67ec0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011249133/warc/cfdd08120a172cc46dfb9b4a8ac6bb93c1e837a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011250185/warc/5cc09ed0081788ebe64a3e1142131ca3a3da1263.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011250349/warc/96835e2c070cbef98f91e53611b23b35e0b2bf44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011250577/warc/e68aac0b96127e242dd45eb44d97cff974266dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011263617/warc/1718ff796264ca036f1500a7896b4ac172ed683c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011267211/warc/a8f9407743e6824317c1749759534fa2de8267de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011269626/warc/9cbc5b6add1884b64a311096b1058805f3f4f1f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011278480/warc/4b32ef85ac28ea4c64b938d73736fbaa496f8759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011278661/warc/95884823f3c47f6d12a4f2b9febe6044d472baaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011284846/warc/d29fe94143ac5dcf8fec6c30471bbcc8a74b6196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011294162/warc/3455a40c1d58807d877a8b986f4031207367dfa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011319537/warc/572c9f79c5285e716556a4d90bfe14850c4a0dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011335666/warc/93d985a7195ea0ef1294947369467acde852d922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011338837/warc/e9f6b5936025091eb449c1edba54bb97f3fa898a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011342618/warc/bfe4fd6409690dd331a47a9ac45908df8fae23f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011352596/warc/77450793a2f23a109527c287fca97b9edb25be01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011355201/warc/3a27b002516c778b0137412f22a853d61444b6a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011372778/warc/c503f2dbffe9f18742ba3ba299cf287476b8eb39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011377581/warc/a838bcf6c2871f96e5512c6c49e01316c51104d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011405327/warc/cd4ef065c9f22bfcee9a27ac35aec9a646b514c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011473737/warc/6f65a5417e87be6aaa04727693c33bb885ce25c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394020561126/warc/b77758a8ba58fe60d331c8bab94f9a1140009f92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394020703260/warc/c1f517e3ceda3f7305798d74c99bec3e54f3bfcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394020792760/warc/0dd4a095dd99675090bc0f7a0b9d1ff5641f5d9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394020986799/warc/92266f1991aa7d9e2485738345f3481e81133673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021083897/warc/a7a7cf85de11991eb4e6b4ccfc9a8ffacca5fafc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021097827/warc/1c6676cea66b33d3d4aff9a83315b325ade84441.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021227262/warc/a70fa875ed9a7239d6abb37ef3433934954c89bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021230991/warc/0a711c1a653731697bc1aea581aca0331a05f3d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021251996/warc/502631b66b12e2e6721c17969b6cb7cbaeb553dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021278114/warc/6c97e5b6a7e4e8d13d6f95f2fde5fee159c56dbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021292989/warc/27e97109b244939084b470bb358230157909714c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021306108/warc/9bdbe2b59e8ad3b468e070c8623a11bf0a74a8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021338216/warc/0971bea1349f90fec252368c4b8d80ec5e06bd7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021342244/warc/94e61b3b99ff5542e9c0f2b30d884b927daa4c57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021365169/warc/b58fe11baa8b5ea0cdf3f9520be73d66d4b8aa04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021378450/warc/0f76b6de3cff48bf90e6c13065f9537ebc4cc0c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021384410/warc/861312e51f394c1d8f0bbcefdddc991cc4931624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021389272/warc/f619b95a146aef2e7582764553db323a289f8d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021400132/warc/6f7fef90e18b9910cc486d8f6b6be018751cd44e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021425440/warc/46395e58ac7adacf7bc5138bedd1e69110aaec4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021429299/warc/8a9d5e751b0595bdafebc38b2577bee744897486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021446249/warc/ab483c4b6fb811b8cbe58dd56a3f26515f7a7da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021453828/warc/bb58037f917e6b0490f92c247b1e3a27327a3a26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021511414/warc/6ed17adae09917f78fc860d3df37530a556077ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021512937/warc/ee465cc4b5077ea643b08060d27a41e7f6a81fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021537160/warc/3b973359fdc80942d8dcaa10096d362c414ab64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021542591/warc/a70e686cdce60ad44bba046f68b3d8ead4d9fd48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021547621/warc/9db2c94693ef99020e8646d5b84163b82aab7954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021585790/warc/409c2f8ec965ae1ad4e98e053e1938e2ff79aee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021586626/warc/aeb7dddd42e801b88081e145d058fe73fe31f1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021587780/warc/2f2a4bcef901e16c707a5e67fba9bcdea22d72b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021643070/warc/604b3d1eb086753c26a5f90be58e0e0be396f236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021675771/warc/f83d26e9afb2c524fb87ff302691f38cb30feb00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021719026/warc/48df1a3212f689723619f8a74d9987aafee1fd33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021721153/warc/1ee00124c22945da1722b6c54d027dc221c3f200.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021724521/warc/9ca027485e49cc68e583feb92baa91c963515fc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021727061/warc/cdc82c528c5e8012836ec9bf843c20be63fb6ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021740974/warc/87d08432db44a75df01548472f843f40756fe6d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021758120/warc/4bee6656e569958dd29e1e560d2d05c828e6e1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021762714/warc/f776314f1be28e5b04375ae9360f43263b529d8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021763647/warc/3a03eabfa5d15cfdc70b05b1cb1f8c96238a1fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021767060/warc/947adfa68c4243a39f5b3fd4510f57fe6d3685c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021767149/warc/ffa59b7878e788edbba47d12f73649c9870ef479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021775214/warc/4a1e2a2d958cdbe85589866276e20a77f36d9629.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021791079/warc/ad821a8e072f965dc1941ea2e94d97b40d81085b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021856395/warc/5d9693d47a64189d8ec7f3e1f5d4cee9e00287de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021866360/warc/dc0755692de39bcb09c8559dde06ee3e8e8eb9bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021872753/warc/b54cb14f5e54701c5835a6702734eb14bdcb3619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021878262/warc/934421f1350ace6461e11d12e7a28f7a18f93a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021889832/warc/6fe1c58c3f48cd7528b32885dc7ba266d0cd0342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021900438/warc/8a4a7b5571020316bdd1fd779f478ba80c8cd6bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021901207/warc/7c5cefa11d86d0629597e4d5c26b76de09940e31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021919409/warc/dcf9dc0456ff65965b4c9167214313f95602b3e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021920399/warc/cab993f2ce2c7f87eb3a3475f04d4d3a95fc91d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021949508/warc/957e3a5882a24f9000a1b433458531c3600e0971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394022296192/warc/235296d5b51034fb0949bf5b3bd91a35fb2b086b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023065135/warc/bfbc049bd7269277f50c8ed65aa2286e58a0e5a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023122061/warc/118d19e18847a049bbd75bd31c5db22396c58e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023862507/warc/02ae40b1dc194a1db1acf4ff5f888c5dc8c3706b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023862701/warc/ab72247881b13c69f023998790c24b474af46ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023864525/warc/246550fcde3dc0bb63ac1fdededfd18f85525204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023864529/warc/3c53b8e98397ba430ac89e17e5cf34e2e4303099.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023864543/warc/a7160af4efa7de6806da8f821289b5b8c64f87d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023864559/warc/0b6733f9a87099bbe395f81b11e5c78d56280bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023864908/warc/52d596b246d76e58da6e1c1f563a826936988062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023865019/warc/5c1c698cbaa53de4bb87bf30812ce5bbb1c98fdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023865238/warc/c8aefe8bef81870b574da14f4aafde2115e1c5fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023924839/warc/ddb58e7d7d50c8fedc655abe4cc5f5a4cf2e7246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394024785290/warc/3ac6d9de57336191d9af542d0d5c808d8fe290bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394024785431/warc/ce6265e6a20f5a7e7a37802dd7811b87b080d557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394024787620/warc/3218c69f9fe495b8432bb9c3759082215688d9c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394026201240/warc/c3d804144dce0c1b65115dddbf0d1f619694985b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394026215078/warc/e9e7406524ff697b2e2c81602b3c89711d9690dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678663724/warc/b17c3baae56df199996a5e60b42cdf5beda17556.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678663927/warc/42584314fb45edf73584cef7edce1e3931bd4303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678663943/warc/bfa5348a5b497e595c2e60e6f7445135c411cb8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678664178/warc/029ab6ffb09b92608648b9b4d537db7f36918a8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678666156/warc/b578b597cefd3b40552cfa8110de79c12cfbc1b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678674071/warc/155809154eb5db516dab2c8c02c41e41726e6850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678676855/warc/7b8dbcf7fc832f8da8cef44164363946411afd3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678677514/warc/6810d8363582dcf74a2746a05f290d8853f83398.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678677569/warc/00964e2a1238e3c6e811f9e099e3d9c2bca86428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678677656/warc/13a07933fc2e9ea546c0741fa0714d41e85492c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678678233/warc/9b1d7a0c1b3acf7c94bc562a9fd4c7b6bb4c52c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678680766/warc/c644a5fd8d67a821ca86870485291ece83dc7968.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678682078/warc/2ec0d89fb6686e266e734635a7310aa1b61608cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678682243/warc/10036889955c8a9d6afd6dc834592ce846e194c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678683052/warc/c29d3a7e25269cec6bc8a87fc02714f27f8a5068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678683400/warc/a594c8bbe23ca78b5c28c5131083d89ac45d8619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678683421/warc/c62619d60bc2db6e24d0b76d0650c97657b15efd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678683543/warc/fbf050e7f4658ac594872a9909151c547108e58f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678683789/warc/2d5f264eea83b26835252536c7bcb513f617c21d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678686979/warc/6f3fd147dcbe020fa46d74f127b1418531b135ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678687395/warc/84294afdda419ae5c2039638f66a6ae0b6ae03c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678690318/warc/44e35d45a671de400dcd43cfd8bd3d1cebb82a41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678691890/warc/b5332ea9e29bf368ea05371228ba1ad4caf53dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678692158/warc/7f7358e51bcc2caf895d8624dd786e7745fbe7e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678692742/warc/f9b592913c124692f542b23773139dd57170edf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678692841/warc/d4545adfb277bcae065a289743fedba0e04d5a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678693008/warc/bc6a8497bb92478beea1ca1c8f56d52bec2afe8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678693350/warc/c0a774c45885db9ec4fea02e62640bb6a20a07d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678693548/warc/67333d6597b1d4e0a96ae34b4493c146850e98ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694108/warc/910cdc1791cc04840f3181a300e05b05e07c2ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694248/warc/c28c66ae92bb74219acf0b1fb1a29eb3276a0254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694619/warc/8f2783bbfeba144b6f10f13085cdf7e5fd8c2a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694628/warc/8ada55127832330090b32a52bc315344961db65f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694630/warc/8a9f11e316ebc8ff966f31bb19c19726221e50ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694885/warc/628a4f0cc590a49412c086804762828d9c0b85ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695499/warc/f1d7fcaff230eb77d45a721b3c0504dc720517cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695509/warc/1d4a1980b25ff56a08bec2da983e426b0fcc196e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695535/warc/9ce6322c51aee957537e41bcbd884ca1525d3c95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695683/warc/0c2a8a5548a594af82867ef20d8948c63501df1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695829/warc/d6d772c5e6c9d5b8b84ea7639b5c46b32912d096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695896/warc/0c66b0d8937f6a544144ffae331d9212ce2134c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678696015/warc/2b22945ce55c570610fdb63acb09a709c0685561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678696502/warc/697e36f65706387fb01c45d50fb41db098a39553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678696864/warc/8c2515b0dbd514ecf0e3ba0397d561ea03456de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678697363/warc/9ed55ace94bd5830632233068571420ae5838ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678697773/warc/9298a9893097ccdce10c47accfea6725dc2dc27d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678697782/warc/18f448b7c412d4a30ddfb65a34c3d03919b5eadd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678697956/warc/ed60b31828c1812b0a4ebf156fe559a29aba0260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678698356/warc/264d8607897541d73db2a6def488d9cf4bf5c082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678698575/warc/262428f243805b721fd84113871d7842b231ac61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678699073/warc/4d5bb3eb3dd15aa43e47334b03409aedb28b480a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678699096/warc/e00126923da8c14a189cf1c04e9a836e17ffaaec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678699570/warc/3c730797a735d7a1410548f28a6a18f84a735502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678699721/warc/559e4812d5f6d5c1dbbc131fef48468d81a3a6dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678700230/warc/b1bd5ebbe48d1d02a2ea97530963ed66262b3ea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678700701/warc/1fdaf1f64aac690b64795618be00d65d0c66a43a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678700738/warc/019de3311938c7be1b161c51ea2596b779dacaea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678700883/warc/45cf2de7b6f1697920083c755b51bda31c9e3c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678701185/warc/b5c19823136ca095083c393200c3a824b8ccd036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678701207/warc/284ba8eff1b6ad0fd6f77a153dca73cefc91a775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678701804/warc/c798ee4f7255975926e404df99f90a0fc6457da9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702045/warc/7a27b41d5c6105e7870b41f9e3e4c9a287e1f6c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702080/warc/581c2f41790fbdc848a7369eab4f3e2bd60aabb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702159/warc/ed1bf18dbe78cb2e50e5114328e49a42e2925cf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702332/warc/9ee0f4da09d370664aa8a9ee2f64b84143bce14b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702437/warc/c8d6fecaec15a4a223824b01fa696596630bd5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702690/warc/5365b03b81111c3a942d101d92609aa3ba611239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702919/warc/9d846c0274150fefb1fe3f868620431a8756ed8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703018/warc/0e368dcb87b8779c639d64979009eebae7341941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703030/warc/b895dedc11503b8de878758ac02539bd0ffcb96e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703273/warc/0db506734bd9df3099b6db2fe8e611f24f2137a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703495/warc/ac058a4ddd196036d7f943b7ef7148ea6b208613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703621/warc/c477613784a29725b0044c48b4bea63992d9df7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703748/warc/22e5087313e0f87893694ebc893d8eb471ca3446.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703964/warc/0549302c73b6051c160f9330ef22c9a795e3f8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704059/warc/7fba24d2082d1efb3141d3924ad25ed19bd202c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704362/warc/02584f42fc4f7e4d2291c15dda3d86f33b7ac97d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704396/warc/3f53e23d141a6263d9fa2a131a4ed9d2ff5aafbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704624/warc/d5f2e7354e10ee71b67acd6ea4cea0b05548399b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704694/warc/637d86721ae6ecbb9e4a9afc09b66b810808c29d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704872/warc/69559cee3c744e23d96cd84510e372d10f03089a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704953/warc/9a07e603c19a4342f2fc5bee137e39fb5068c9c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704980/warc/2785319b3958400d022e076a6e512d6709abe555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705051/warc/6d83707ca218877651f74ad28d15bea936ea7e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705117/warc/a6dba2f804395df730aaa50f847b5788825b3a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705235/warc/14d6a63c1ad2db1901ac8f6bd89d8410b4ad8883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705611/warc/4015b94ff7969e05c15835ec46a58d0f5f757ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705728/warc/486fdfbbb410bcdced4c85753a1d6c3d8cb02c8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705742/warc/d39b1af34abad8ea23791ccc422b230a26f206b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705763/warc/9fe863a4c14516ac0b55912a713b0c6c8c5f3ae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705768/warc/65e0961086767dcc19326abe00b336a00e8558d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705901/warc/3fbf51dbbe828cc16eec4d3cf214ba2e5bde98f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678706176/warc/14fa103f0168aac43330ff536da7c458bc56e9cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678706211/warc/641571b92dd95043d223142300e02a9cd3935930.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2014-10/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2014-10/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2014-10/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2014-15.yaml b/configs/cccc/dedupe-url/CC-MAIN-2014-15.yaml new file mode 100644 index 00000000..d1235f60 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2014-15.yaml @@ -0,0 +1,486 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/0c8044ccb91088b3bb26d1cf604487d580a497ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/0c962080ed10278cd429960736cbba34a4ef82f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/24749ddbe8cce5b3b3db54e66834dcb8634e2933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/82f7fc2771d93a62edec3f826bf10019d1bc0939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/94ccad95a580314dc8b6822b3379f9cc4c51ee3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/bb2c824115ab552866e7fd21c1fbf251c77e00f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/cd50980a411ac2387ee7938c4a76a3cfb9da5075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/d3f5a7f611f4343bf4fcf9f20632c38177544f94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/6daf5c0f325407a228e94324722db201abbfa119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/860f74f37a5affb538a78ec2854a235962482b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/c04b759698143ef57fda89d7a8d8533785a65e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/ca8a7f280b2e55489c0978d592b21e9e1b44a35b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/cd2b84b66db12481d5692dd2b7a8942538082d71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/e7c953fdf6fe8783812a502587d0e0188dbb43e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523265.25/warc/389bef0c4e429bdb726c340fda95930875f352ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523265.25/warc/ba1807f4c436547f6cb24dd7283ccfac12c9f37b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523265.25/warc/e8b1c8574bfc1cfd06a293277d1a5c0cbcc7477c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523265.25/warc/ef88dce71cc7389a3020b4ea832b093fcfa023a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523265.25/warc/f6872d3285698e9eab9ebfb77ce68e8124af5e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/135a3dea6e13d57ef3c9445e50e39c746dda994c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/2ed705cc02c0ac4358830f02a744a02058e920dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/30452c6c4063bef0b6a68b47006dc4b5ff0aa76e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/5b7e61b64081b712991e2de9da883437f85bf810.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/806b774e4e36e56e4af4e24f3e5ca9e74899b492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/fe17905fe013469324378351597f1eeba003eb59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/3c04509060172d8d340c622beb7524b72f51490d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/72a2e8404227d2725c1122bd4a67082be52ffde9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/af97593db3febd78780573f636f437953a47a1db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/d3ad15680a3b981efa0de552d31fbf6060fed514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/d576bb540d618f2d8f3c310ee1be054e0d975068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/d8a33e43a7b47e60350f7b5f01f532b29c995a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524644.38/warc/3026b8875184eabde498c5f3a8dcc693f28b2126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524644.38/warc/416f6be8a33944c096bc8bc0baaea4d87a534d90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524644.38/warc/748b083fe84d3a9c9aa599b60fb076ff1cda01cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524644.38/warc/bf272096dd0fe0349da0fab6e69c2b1f9557fc21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524644.38/warc/fac3cb7253b2fc634b20883fdc0f0571aaef9408.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/192a8f69cebfd36015e696be7bc43c8dd6732eaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/7fed4c9152447b4ec9fde37de2cbb3b6086c9bed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/8dfc850082b4d07c052a2162a958511a57990d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/94c8abfda6cb94648b0e3583d47b4f12a4eab89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/d5ca1661d7a6080b33febd9bd0980533a83b7f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/eef34040c9011f58b6774d450bfa4d66f00399f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/374286b489a370d40ff58602697ee9a6ff986704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/391602e07398b5ae10520042da520b6fc7d0104a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/523e0458c306a21176d56111bcc123eb9dd9f656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/855ba7f6cef1e3b90ebf1b8fa6b27d48ec66d019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/a39178ec4b3a944ea6347e3a55085396ffe5db26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/c43b27f9ac1b484faf547d59d3914bfc489a0a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526252.40/warc/0af30d0c0d516c6f0f2db7064102659a1adf9f78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526252.40/warc/6855edeb65f4afa0406962bf778a57e09097819c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526252.40/warc/7bfba12159119d57b89c9db0d6bd6ab24978cd43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526252.40/warc/800e78258d2e80817b8723c6586e2201978743f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526252.40/warc/e5a693ca150aa0c7cec930874dece5431c9b45ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/1fe02a955a48447a754efbe52c6b6b5f89ac5a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/70fa9bc64fff650540f89691f2c338f35850a9e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/8395ee7d6f2e03a4de157617be9778d192a34ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/be4755f8b69852a0a4759bdac005b974b990b4a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/ec396b2501c07ef99a40034bc959b2beb0d953fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/f0ddc706ff81ca6f6c6cdc4001f7f3d46334c476.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/31100d647642b5cac3459c6cd081d113fba493d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/651ae0cd807754976490387d57e424a819eca3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/92ff4be5c790ae83a21fbbdf90ca1042f83b4092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/d007c43f50254ad142518e29e4a91f07252ff57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/e0de8aa986d543eff5f75b286034583146d3f834.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/fbb39d7034648707e4dd65a2d3ac7b9aa31097e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530131.27/warc/129d96a88fa4790372aa17e8fe3b371666822698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530131.27/warc/14954a96e680a8065192b46f074c8ad297757fb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530131.27/warc/6bd83cb1258431c7d05cd9ae1e685627781f09ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530131.27/warc/7720d172c97db279a1fb12efb861108a092c3985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530131.27/warc/9bd1cc655462ed0a3748bd2238c7d7e749f66543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/463bf467c0f8e170c8965919f6c621697ee5a100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/59c6ac155c2ba7ec0c294712dbdc688d8d8a0b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/6a2b0de63ba9a049e8d61caf488cb72ce2dbc584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/72de8c0ec6c15b7e2bed0dbecc4ae73c0547870f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/7878a3aa26e610698357f298d1b0e975c83e1528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/7c4c1d1eed12d6af2c4b93648ec2c8fc79ada363.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530895.48/warc/04b3c6285aaff9f86cca9aa6da2189004543cc6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530895.48/warc/24c5090eaa454e3bd5d1e70a52809d254b6cb5f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530895.48/warc/a4cc40c5ec0a39aacfd355b7d871b617ebd782ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530895.48/warc/ec610fb29ab9ca663ff3dc524b5d46f26645940f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530895.48/warc/fe196049ef9c809d54c1f50fea0fce119f9632eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/1bd5bc8b79918b60a3691888aca86b6c4325b894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/4441069a145bdf7d2e95c5c71d88c1ae14e56b05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/6045d09dd85c1bb767af3c2f6cfa1dbbd6093919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/778dea89eb28cb0f06aecba5ba39adec48fa88e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/a737ae4328dfd70fff1bc147fd4b43fa0d9f9732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/d03ac3b3302f04246b848d4f8d9da5a9c02293ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/1e9f8612ce838825ed58e167e9380a49e4023df3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/2b8a83ccb3e5d843f9d9925361b19a41101ecaed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/4e09e62b7adc9a778909d02afcf490c720219857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/6d1baf910b659ca527f797a2b2bd46ff0523d58a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/81323725b8d1280ea89dfee1ad4d28b23e6ff022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/c62b125da352b89be8d06c141defbcc8e9bcda8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532480.36/warc/0fb04a0f3936374030af18f82cc625a09ef6ca9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532480.36/warc/36c7e6aeb0b9c31368e1615949e8896e567e3ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532480.36/warc/4709e32718843d0bc5a0dad97bda9397a5a67ac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532480.36/warc/8b64319b4e2dd993b98ecfcda88abe5f2003a5a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532480.36/warc/e9223261e6c6fdfb0c9d7fc3901339700bf354c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/08f60ae27801af588e5324680997d140d2fc1f7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/420043c03efbf894651ac97baa0234f49bda4231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/75a78ab28e33b03605fd107f3e338c4ca0a63326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/af745d02acce0c7afbb34fe826be951d20852b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/b4169ad8a7e98169785a81822c4febc5b17043e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/d6a344227810eb1a04ba04ab2ee8bce89f7bcbe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/502d6c1e690069c8d788363340e02f71698f0b04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/5ba476eef4bf0bf0864d013c1422526fbaab771f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/76237080a86de16e3c3f43dbfb5287af8af744c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/ba131cda1630b0b843b9e6d7d99f334aff6c92a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/ca69d3f1716386586fd5e2de22df389f943958e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/cc3979bafbb6305f9a275488887b98dc64d64d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533308.11/warc/4e3bb31cae673f54b3a67360c8df19b6d7085048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533308.11/warc/66140efc5421a803424b3bae9287be85ac9e4d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533308.11/warc/796a0931c8efa08ebc90712cd986e4ecaf4dbc6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533308.11/warc/bad8faf05e54b0d997779045a264d8a0d3e6f086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533308.11/warc/d212fedf7153e4e0d4308f922cedf2fa9595121d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/1efcd0d650b7524cf792e10a32792e6af7dcfbb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/26842de9134f257ddd4cf77366d8a8cbe225277e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/56fec9b3d222f1c337e8f5d7252c23f33b4b547c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/cf69218cfb6b4c771dcaf36a58d3e78b2771e9ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/d4374885055a1a65e7e80340628f14f3ef4dc380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/e168db08da11d2e195468e8da5b09669f8a228b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/51b0dd5b25c5d42fc627cdfa396ece2c861487e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/67f235d6fdc863315d68b59b27b3da95ecbe7655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/7b1fb1e8c784cb5b0fe877b3fb91153628364dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/9636ef3d532734102bc40477dc20faf81937fc63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/cbffe4b0f46c7daa85f82f0b1eb2b0e630802ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/cd12b7d4de0701beba95f1c18b2313f03826b88d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.7/warc/7becc837894984b24570017ba8ffd65da72f685f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.7/warc/a2e4e24a14f2f5a739f8766d77464c96f0d412bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.7/warc/a9ca3ac6b85618a77be1cb89a564b215c4579b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.7/warc/b3958c5b73d99fcb42626861c08641f761e0a25f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.7/warc/f17f26f6cd6ee9d990846fe082c3eb9dacdfbdca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/125d4d75abf92c68f57b4315d3958d976dfa5be8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/280b4325d55bb36f967da601837ed142b0b0685d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/6e7da7f0d411dc270bced2fec00a1939ce37b789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/6fe07f567d3d64f8221f477001666cf2d1c38622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/b470cbe05b3071b2f7c32732bc8824bbf6d6e6d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/bffe85370321e4b38e4a7d43bcc6766440ec9f93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/2feb892e7665182d2f0f9f285b7bd0143a419c72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/4208647a03cfa4e42a1591ab0ebdfb3e7b720691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/48a83ae08eefcff37d8abc4987010700bc58f758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/4ff87d1b550b7373b1ca7c98d556fe71d38d3996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/655c2a8627b061893bab972f71cd9a3510bd8e3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/aaacac9bcc7e2bc781ce3484cc4a7b17f1bde76a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/3436a551f2eea9ad8a5c534a33e529a0ab453844.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/5529a6c86d3f71d8de9f9fcf7b25fcbc3da2db8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/5e6b3ae46fbca5f82f1a944b198d3d442cb726f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/62cb5f5de17c50678485e7ddf1c3942918265d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/9ad568624c89d44ed5e306a0b2e54a60309e56ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/f43cb405fdf3d2606c44e6298033dfa0d2dba241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/53deff0f0fc515ffbed9eea6207ad5a2842886c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/5e0ff9fd90e13e21b6dea5a00b951b03b54895a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/822fdc19855e7d89be37be5b52429bd3134bd95c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/87d99de6190fd3a1487b0c5f38ed48d7424bed1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/a258f745f09bda93831e4da9bf72eb7275d25bc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/d557fee1e7345283ef2da800f61fa8a6f7f72807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609536300.49/warc/6d8ab6b2ac303199c9e63248e6cccfbfbd229e08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609536300.49/warc/80f8c892c4d87c89eb173e1122f14ea930b21ed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609536300.49/warc/a6cf8eab20081ade0f51fdaeaf2574f26f2dadf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609536300.49/warc/a79de8eb6c9cd4e1ee391be0cd37330adaf5d502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609536300.49/warc/f43dba93cf5b268c6ec56e71055777a132aa03a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/013e44ac4f6557c7bcb33517b90438bd9ab3e4c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/11318b06bdf79efcaeb272dff6a6b74b9ab9df98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/6aa23945f2f727d1cba1e576d5eaf62e1bf62ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/7fdd4fcd249a664fbed1801fe3a5f0e5d086f338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/89b12f7c28db3c6ac20b36ad55e7096582e847c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/daf6453be9588142ba7d46419fbc45c03c26e7e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/3b3176629a6e78dd3fb852968fd853dbfe630ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/4bcb66c5f0f50cde4da6712039bd8730e348c45f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/690ce4b767c570c2d8f64dc0e618558657745684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/a2a2aa3797064edc2cc0d9ad10f2aafd60e3881c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/c62161c3e45beebd47dd7f18c2d4503197768b29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/e0becf47ac0deb85f8c88fd79519721fdb192200.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537271.8/warc/023924d23f55eaa0511780973bdb91687b42762e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537271.8/warc/133c72be5f0006e0c2af8a20ec25b04186bd5db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537271.8/warc/2b0264f833d83a174b8e593921c59f0d1781d96e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537271.8/warc/489b9a34c5a7065dc767ddbec3a213ed8e1feb17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537271.8/warc/5440d3bef5c20c2f333093c80491a54e914a4ab7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/51c0b541af4cfb39fa14d6cafb66db10da959d21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/965ebdc52993c6bb7e214b05916982f1f143f7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/b3223d4137a7fdeafff7253a65f651a8556254bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/b7e1704dc7be3508c1762ac42bc57ef3da70d6c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/bec95cc260037c9548f1f5fe5dbc4730b8f57ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/f13eb5176947fa7f5ebdaade971d8d6abd46f3a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/0f43a5df7fa2fafdfaaf6c4d44ae687238801177.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/4461561201e3b0dbdbf9cf7d26f998d2ae10f82e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/702122264bc4ed199c55e86d34012760569c7ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/7a84e4be22f8af5cc6adc654584082c6eab1d49b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/817b2267534f0a113ad3cb3fc9468ff52698c06d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/b5b2b519a9b9cbf2ec8b70390bd6e16e657aff84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537754.12/warc/110fa7aa7ead46062660da8b427df1f28cef8098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537754.12/warc/271e4fea74164441bf8f1886b3372cd816dc44cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537754.12/warc/52e868874125c7052a220f6a74c02ce11f2b54a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537754.12/warc/5372b86d6483f4ead16194ddd5bae138fd2b11df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537754.12/warc/6767dc5424705e0f2dd5c82525021ea9533fae4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/42de27261d441dacf8674b2e150672fad3af9daf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/5f7db9ef15759bd6916d31fe749b2603e0061767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/65f3e0f974da9ac348ccc5cfca0e876591a8747c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/78d3d1b06ef0d2e2e61a0ef41ac1262ef68ad713.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/a100783bbf4145d025e615aeecfbabb5e357ff3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/a5194a0e2f619ea6f114eeb33f1ea988eae1ace0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/040f4f64ca43a6fb3cb4917ca061c675d7fcb920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/58bc29fd26637dd4e1ecbe8c09de738858278fa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/89b3eb36c426c2c03f90fa7fa3f8ee36893b61e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/ceb272d2b1b29dea268f911df16ddb52ddcce2b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/d23971ba8ba642a0dde5e3c7d262b0d4777aee5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/dae7ff85b26528c8b4121df07c6b5b4cb60e7103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538022.19/warc/8aff9b1f76bbd4ddb15f6e3a5c8b8c3c2ed9933d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538022.19/warc/8b82e1a79d33ab431a090e57785a88d127e30635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538022.19/warc/e13b653d07ff7acfd6e82bcb7186d4a959a09e4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538022.19/warc/e1edb778b95d50f99294f4baeb4afc0b91e7baa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538022.19/warc/eba421ec01e0b5bb345dc3dfe4fb5695b91578d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/3ab32bfe40eb694f568c6e76b25e4db85def61c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/8270810a6ee0165ce6269468abb7584a9ed32089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/8a56db8dc8ae824a4895748778a6c97389213b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/a58cccd5da401a6679820d61fe073c5be482abd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/b8b1f6f83ff3b77af28aebda8b1d086759f47039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/f4c042593048ee8c26753553482781bb71660553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538423.10/warc/2c34e71da9d9e4210774bdd30991c4443f3ff13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538423.10/warc/7adfb6bc8b69f64e499fbfe70f5a130b4d009abd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538423.10/warc/b28afa731f3bb5711b0ac12a6e014a2d072b2994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538423.10/warc/c664a3d384003b6dc16233db1c8131a9d1f93550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538423.10/warc/fdc48d2cd5ce6e8eecae272061cc9c2de1002ebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/36d7041527d9ec92592d242ffc962f4cad7e4c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/535c9be78fd6fc6b0fdbf9bb9569e2130ea59fd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/7d63a1a68500fb97eb1ae5faecbd6eb18d6ee2c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/b0d9dbe3b6dcaf617dd31caf929478f5685f9fa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/fe11c705ca084760828598f002cd9ebabd94838e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/ff8d408126e71bea8af7b50777088807ce71b1e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/69d47e2f904897498769af40a08101660f18c25a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/972eac4e28f4c1f65f01c9cc983a7b82dc0d5d78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/a13eff68f49b60724e394a31c48eaca2f22f766a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/affdb06bc379f46cc93dc07d5e3872941722adf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/c462df248642a0786567ce1ec6d51b2dc99e468a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/fe20737efb1e65849799f183c3c2266006569d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539066.13/warc/0febe4456da0b57e45b32160dd6cc56015f6f317.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539066.13/warc/5f095128e3bf933e43cd543ca73a1a968c672ec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539066.13/warc/92fe87de4107b35c216ecfecfe071ed66587619e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539066.13/warc/b31aff080fa0fc90f71925620a6899bdf86a0f29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539066.13/warc/c4c5107140d2428d7e153c80d8e0d2dfb894b83f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/115952d34671a374b26d4a5619707c8596d76cdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/6015de326cb41c819dcad351bc0732c4a81aec9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/7b43fa2fad1f4c34e401b4fc252a97a24b1cb1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/87585c3221975fe2e00f9c07b880c34ea5c87287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/9b0536bd4f050a2a3314cb44a0aaca9f2dd03b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/d0fa3d632c5e99bba308f2c602915f2110968b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/005c52d91a747dd89b6371f49c259d5432d4ab4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/28a802e809fd54b94f4878933621d050512f06e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/892ce3ddbeee068fb38882ee087bdfa27bd2938e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/98c170bf21c1e8893c6e23186e640ff775c78284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/bb912124628bb5676f9fe2b9c96786f1bea74b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/f9b09fbb5b858f78fd79ecff6c4bdea5dfa6666f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539447.23/warc/2c4b774f9f8eee27b6bad196dfa65d5e68e037b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539447.23/warc/7a1e61e14f69b335a0d9713741bc8d30c2740dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539447.23/warc/86a6ef9418430a086a2fb23dbb6ab4ab57343468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539447.23/warc/c781bded5eed7dde355a328229a2907226b77e91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539447.23/warc/da4f811e206eb7e93e0d62bbb0c751d4e27d57cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/1130cc406c6bbbdf48ed2efab35e5aa81145d39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/4eb88bf69ff317e3596f43539a688de65baa7746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/53a6d28ae03bb1973cf6a8c514a8b7ad72f2f08f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/ca74f94dc19e4e20814c650ce984c17eb043d19a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/f6270a91b8bcfddb8ffd678b56238e17352aa7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/fd99d83d584e069c283d3fd5c7e2814a7a8787a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/31397753af6eabcf5df1662ed6791a13bd591ff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/3ca0b3cfb6fc396e263d4a4535616712d509cc03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/7625d7e50994c7ce270eba25a08b68d5bc265d18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/8aa66f72a507c5cc30df8cdc3395802dbbbf6fcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/8d69ed019d23ea616a9bad0fc678917f028a220e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/e13e01468d49c22599c181042a03b689db98c8d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539705.42/warc/4594c21d6b40b10d8a0ab00ac51c9ece886265ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539705.42/warc/5c54546e03f310768ae2e0289dba83945137a1a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539705.42/warc/5d258e53c490ad4e99a9a88b45257190aa18f02d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539705.42/warc/7b203d30556bdd1ff370d4ffdf3f7ddd13e2c77f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539705.42/warc/97b153d1dc9bbe6afe46d642eb69cdc7a3584266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/1d7654608a0416a13cc77450d60fa6988c221a41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/31ed3354a9a72521272b9f60d87a3aed6393a13f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/38d3f179d1e5ea5d821d1e754a945d1745a60574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/5ec40a328fcf11bf01388092d8b4c9585164cace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/72bd1eda23f7f3a1204962e2fc533e56254f644a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/922e6cff55b5b0bdff10bca7f0665f01716474b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/8772097f67fa84b19542efe87aa8b586ef0d4f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/8a366cd353bf9c95697b582b1568ac72fb692ca7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/8fa5f02634e6b266bc7636940205660387147561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/9208a7cd54d93efedb743f8c3f8b5f0e71ac3ef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/b571de706f4cb1e20dcf9d4e9a541267867bc8a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/c838801f9e5fe8529d10a8372bed6c3f65eb6648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/3c60cca8d7835015619ab0540118f9d929bb176a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/7126d6bd9ca46b53e1558dbaaa3c9ef205d2df42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/936ca79c87b6d8df3d6460eec444c384829f3290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/b1b4a79707d0edff4c53af16d7c9da8a92211a82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/cdf6b8cf5378f5242b35d464118620c61cfb3048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/dfbd5ca8e8517933cea3388523c8bc9077b117b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/42f4bb312c19526510a67487494a6cccee91a8e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/bf4424504681548a407742a90da2d53fc1859114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/d12288b7029aa308dd78250f44b445508ed82e99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/f26566a3cd07b7be5f5b49ba29dd92bb033fde3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/f5039bf4959b6ff8bf2c8c410f5f649280007f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/f91f76739be8ccc5ee02709a56b5975105a8d433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202548.14/warc/0c5c572973ff81f810663f696781af779e489d4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202548.14/warc/43480d07a9dda92ebfacab37afb80345273f5379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202548.14/warc/64f79d29b22cbe6be8f278253b12f75bfd039897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202548.14/warc/bd1916844e3b1a3a0cb48469d5c6b914f74c1f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202548.14/warc/c9ca6019fdcd577af9f661b4857e632784e3e799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/0180f8258718c1dabdd12fb59717471697089d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/293263f6976a28a8a68a965d46db00ced09f932e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/4184ebee90b345e5cbf166991fa4f54791920985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/954f8a5c2e30ac0bdadb1fb25b6bc7f0b9ec6da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/a33cf68940f3ab8327911e31d70df2abfcec268a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/c991212ff28321fad240879dee263ad2bfbbd4e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/366ec4bee156ff88d82b72251f01e87dc0eb2508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/656555e95fc8482a3126cdf82c06592ad8cd9a90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/ae1f128fec92807881954173ec866cff28c466a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/e16c1b68d61d00e5f3419839b5bff433cdd4331d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/e9b80780da952f3a8c26af5ab7799655b2241fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/f493711bacae0d6a13363140b9be180bc4a47e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203422.8/warc/7891f636e8c9c420be24ed5d62be36f2dfe4bacb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203422.8/warc/a1d900f8402024bac6c71d36d2966e79b5ad15d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203422.8/warc/b2c6abd978e36f032e8cb53dad4339332239bf06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203422.8/warc/c5eb00856f235d78a4698fdf43776b83400c9f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203422.8/warc/d846e6e822e27353fb5e414cbe6010bc3eaffd83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/589d4c5265358cae2767783060e2af3f578d026b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/590473a63057aa9d9206d4fab89e8b7fc2a87746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/7806f0e1236545619bdf175e42c929d463d4cb42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/785fb4b9b215c33e6d59ef4662b8f2085ce40ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/7d9c1d4c27085c276dec80d40e9c249b1765271b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/e3a354d19a91e4a41f679704ef66dfee191c94fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/376c52a9d5dc65d217d1de0a50c9b5df112204d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/3c71332d9bfaab883775fba4e8d6a9645c64cda1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/8d75da306e97ce256f49220b00e8021e18ae5b91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/8f1a8e55a1d427897c5f08cb6bf06558e5f2e950.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/b665043a2d4ac7f3b90d4f1a6ea279cb005754d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/d46ee581206f7a6e531162c202a8bb318ef1aad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205137.4/warc/0c0d1cb3c79bf9d9c3f4048dd86b8b724846af3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205137.4/warc/0f97d8ae9b17c729aca1eaa5a417498c486f57ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205137.4/warc/4e11fb6548f060bc43bde525d31c7b9b20f082ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205137.4/warc/9c07b62d7645fe0e91383c6f8f7ab9b20402cc74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205137.4/warc/e07a2bf480527399c53e2b6ecb080e1b080d295f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/5e72e764183116c6f18a854cc7f4139dee2df693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/74e541653646121a607e2272ed1b413b30cff973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/8bbbd5eb9165e5bbc328351654ee0870122d74d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/dcc5d6384bb6de2f64be1da7e257e4568961819e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/ec24d415604638c085ad1a53519ee2b6f87e489c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/ef2c5685991e7e15a221f61198ec5f8ce66b6992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/07cc1e284ca6f6ec709b49a45c98d788581f79c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/0e38582466ba03849a062d4d6633b21bc7967e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/5c7d60694b7f9a2c63309321f1c3be57a0c4edff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/948eb4b96609a18072eccda9e715882d83f813ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/b7afd65c95c36de06b417ef43a7b7157fe209913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/fd88449824cf0720bc94bf8a33efda0874d87820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206120.9/warc/2b23b02f853bf2ef1ea1ad912d5d1b5245c9c91a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206120.9/warc/438d66759f0242fb766cf7bd0543cb3e564edb28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206120.9/warc/5d4998b72d4883b874e353a004e7ab7aefb29887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206120.9/warc/c9d753698d38927ac2cac7341fb92aae5119e444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206120.9/warc/e013dbe2a6b36ac9b48d9d8249a680139240534c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/ac7b40279bd48f23462f8cb2a1176d093a2b6192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/b4112647477bd0d94116977a05babd2ef1a9fefc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/d188d63f545b18d61fb5cfa4e0845f4821f68d7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/e536d8c7658059a4af308c1c1fdfc16e03ac49d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/e805d54ccb618b5dcc396de4aa0cb412b2be492c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/f032acff2074407550e592c37a4d27b2d7222ff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206647.11/warc/3b73c075821e74ec2b5f750e4feb6e3f2b9d15c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206647.11/warc/6a2151812cdd3e5c61966310c92f73c24f5fedd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206647.11/warc/6d9a92585a700c3fd79548f787746b979b006131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206647.11/warc/954b7793114f9920b42ada01bc85c548374b3636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206647.11/warc/a6b847b6983203ac3e6879752fdfa443310c556b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/10a4eff1d00eae558d147509cb20e626e44cdbe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/7b09d71173e3344b6b2620f819202836faf70c47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/7d3c6248104364a741215b66fedf97c4fa9e1862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/9a2b62fadfa7ccaca7a1575de59db8737aa1dc53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/b8bbf01a277687a33ba74ef637b2655f02d162d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/e9a1e9d91c0df0e918d2039a04817f988576b196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/15391c49da50c77f1ecf1fd8b65cc1d18e931aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/b11d8e8c15caa9806730787806c9e222fd7e347e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/cee19c730369b5fc87b9eadc7502e1aa88c0acd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/dbdbe61691bbfcfa4a2e54e2cf190651749724cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/dc3c8ca91e88a5006cef3ec32361e884f83d1587.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/e4e7822381a7fd23d331f740c0c02f6f9195c941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207046.13/warc/104580de0e8789449e323f0a49e4ba2971097a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207046.13/warc/2d9efe04b7fe66e3a2b4ba2bcae6fa11782ce062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207046.13/warc/96da3cbebc57ee57c754e64ab9d556094c772ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207046.13/warc/bdd101b8d26a882198d9a8ae3f490d5b4a1682e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207046.13/warc/f1fd0f1d5dab2aaa575c04e6e6488a998b70b80f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/23d33a5db53b7f4c53039e97e110e3c82a9ef8b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/52a6c4df1200ed8c5d248493c4936893ee408b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/bfda8c683f4dee08e5d2dd2a52e1b271be7cfacc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/c3675eac21849b024ad4ac0b8a4667431782af95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/c919b661636d11806cbd3435e3dc7584b7507ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/ee9b791f60d3f602ceeebfb5a04c25745f9a35e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/0f87f63efaced70bf039cf74725396bcd85e595c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/4c32c614801bf5b68228464962ef0c104cd305c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/6e0f66de14671e7b502202952883f427646675db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/964dfe301b5b0b74d10fcc3db095d748186f22e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/abdd6a911b0d052cff387b576caa08738e2b9a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/f799c9adbb6b042a348a28212eb4af9e5a4ecdb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/1442febeaa4a786eac7ba92f048823ff836cc43a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/34cb6f2991dea5f309dc48dd49554aee26fb8fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/48ee3d1f296539f9df6437dd8b693fe5bde5601d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/5a582835eda1d58f7b0a9813ab34ed940f5dc5cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/d4b0c3c47272210e19c30fa44e0f8951658bb695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/db0375d88bf8784e6b12dacbef444116b6e86276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/13a4869b0b5d7375158d916b259b71a766655748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/16c00660f4bc5b4b18c5f4f5ce10e40b1d0971a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1abfe1293bc3f253ee1f885d60862ec63e72c153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/2174e0f2df23053164403e3c66415c4f0e161966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/227c140052f660772eac4ff68410d04439b93e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/23d5642cfa96b01c7a786251f697c56be5ca8a5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/2414a66b10c812207fe9d2468acdf6dd0ba19d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/29de245a746572c861a3340aaabf7ef4620371df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/326d4b042a4e64afffd6ce006c348c9134755e5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/33076f080443e16442c94e318ea978e7e29618e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/3424f5e63761bdc9ea5230ba9407210c94fa0454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/367f99e4bfd6e44658f8d0dda054c6f6c21e6d24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/3b18a115232f4fb3691ab0ecaa08be2aac4baa36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/3d8f4bf59fb47dc9bf904da9a60506f89459f606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/3fa6ce4108d2f8b773970d0f8bb1785a551b91e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/43bb1dc07f8b9996e3700cc979b0543fef095ed6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/447843e0df94bb943b55111301fc0870021703af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/489973e2ee276c1592a68dc3663c1fad1089f5da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/4a2439c788cd33fadcbe93acb26f8662f1a5d758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/4acbe66ac9e61cf04db4745f3ddb7bc31fdf5d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/4b0f72b143709805bc540403bc9e6a67aeef0e66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/4b58a2f50ef1ea4d2b54547e7bc81ab469f3248b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/4da7c21ff79990ff6751cd21b4ed55f8585fca48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/53d19e7cac8b02a326967c70ebdb4264563f7a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/55ac55438a6bd4aac3cb3a5b7d9222f699c03542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/57848cf16aefdb1e90391a7906f8329c9d8d6134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/59d8c2f83825ffca51ecf7f32a16c0b191ad86bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/5b02195b744f7b6d9d337fa263984b8980946e6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/5fda67768aa089c42ef8d7be8540ec91b23924a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/66c3520466edc18db8c14e6e72b7dcc671b968d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/672f6d4c1bbb8311411a69d3e6d49a881ed33d0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/6a5525587d352dccaad55dba3c82b216bbc0dfa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/6cb072a00145efa973592cbe4e3c4360a6e5947a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/6edf58f4e509d0365522eed007daeed1967e628d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/7582814d760baaaa6540d0d9eba6748264120300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/805162259ad3601314b2c80bf4498c2abeb13302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/88425f901e9ef679d6a310baa62936bc90face02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/8e9f43bfb9405a717540049954120b498e68d4ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/9512e48b9d7a872416699676b93a7708bd3b5c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/9948d860fa3c05676d125480395a3cfd13af33c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/9d747cd1ab734ff0918f0a087d1613def9fc0242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/9ff95a95a5b25d808fbab2eecee7708744586c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/a13aa5d139735f0ee9ae7d0f3862958624398dd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/a32347ecadc44ce8a945c85d8830b828ea9cd8cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/a4b84e5eebccee73923a7531b7ce3c17720cc144.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/ab8a4bf8daaea4f028509c0e9dc97fd53042e922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/aba2096d2672d64affa364328e0b1f16fa73f3c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/ac284c28ad78fe1f7f50bc42b286d10842f1cd3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/acb1b3f2631de480a2328db3b6519f6619009f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/acbb06a18a2523289dd23c0b95618413a3301e31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/af802d7e8cff890fc2b10b286cf3328d1dcd108e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/b4972cb7334e55482b2401af1ea433f554c652ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/b66625ab9cdd4e64fae81f9b168b58f480c213da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/bd6257d9be58c6c2a7acabfee80172a5ef53dff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/bea0133e03dace47d02198ec7e78a9fe8f7a0724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/c8e721db8b111fd6bd28516ee38f9f55d0a9035b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/cee644a8a3102417db7dddd9c3c2e655e6bec15f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/d0ad6489e37509cc723ea667a3c202da3c3ea51b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/d84f86ffb135ba7de6f119a00d25fd52b6065e46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/d93ffea777be04be929afd1bc4aedcb2c45843be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/ea41bab5c94463d7b3c51c587045f593bad4ee75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/ecc66985b17ff02e701a08502fe1609cf7b9cd53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/f092b3f80f9a8af1e4844ef06f499510b9538999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/f184d09f1cd867c08b5f603159132ab249a58e06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/fd1e1045e6c57ac449b05578f26de1ba5769bc08.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2014-15/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2014-15/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2014-15/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2014-23.yaml b/configs/cccc/dedupe-url/CC-MAIN-2014-23.yaml new file mode 100644 index 00000000..6efb11ff --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2014-23.yaml @@ -0,0 +1,657 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/015e2f39115fec71e47da0eba52936304ed1c87b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/01b6a13c684d379145994dfdbc05b2f14821e438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/031df5e27a58423084b0e5a420635d5d0c2b0067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/03d921777802962aa1eed763364187511f0465bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/04efa16d74b25973bdf082498d6e4cae67407b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/0718f4d844d02b32748499467bd29e664bcde9b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/07ff2f3c368ca01f4ddff90bc88a2b596e6a342b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/09957787cf46808f2df368cc3ef78f789e2368eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/0a8c252113a2fdafcbde0490210d66d4f3b6afe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/0c858d5e384d1fef58c3988ddfe56f332a7ca341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/12ff01bf30156f6ae84e64fff6980cec85420eb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776400583.60/warc/588c1565536a8fb93d81ba0291b3e753cf78235f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776400808.24/warc/198a49335f6b5f42dc7a832590cf017ece6fab2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776401292.48/warc/b541e34398656a1f10fba53b48e761770c48ac92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776401658.57/warc/d31c1d2127f18981cc603cb23da033caa676119e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776401705.59/warc/e6c5d2f37f2b9c42387d12f44c4a6506c00574f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776404630.61/warc/4e4cc8227855d5b2439a1a54c5fa584d94de4b5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776405824.29/warc/d177ff23192f9bcc633d2ddcad342d359f991cd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776407319.89/warc/24ec32b87f81e6ab3a4b45c1ff33b40aee39c4db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776413052.23/warc/d86f1b2fadafb211953046298ed7d61b4757eecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776415016.1/warc/c2b57818241d66d13783ed40f43e0a4544b0812d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776416489.79/warc/f1bf7781c4612ab4eb33612659964e01985ba844.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776417380.9/warc/80b2465d0afa5e17802f1666865827fa3b7c3cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776419693.66/warc/a3f68f4de4efebcaf92f5761947a74a41b45de67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776420123.16/warc/10af17a1308856ec2f65985bebf91549ddce9328.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776420526.72/warc/bcdc7fa4d5c99d4c167a6de5d5d52d7512a523a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776420978.98/warc/5c2ec471d9882ddf0cbfa59fe760cd814f95238e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776421646.0/warc/cf24fe742dc831cf566844c6b5f3abba14b77107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776421879.69/warc/daa68e7c2f856511574e9421840a408d1a0f4ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776422137.22/warc/3ac4aa394b2e13ac332bdd6f7a36156411448dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776422751.47/warc/0df3d67fd0d94bd5dd6a5f13c1464053788d7bd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776423075.13/warc/31d8c065b67c389d7ed5854ffe7b23b75379a3da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776423684.4/warc/7fe26e4240f0972d26c3450228fa2fc756de3a7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776424328.95/warc/d870450d95a8d102351bf8ceb3102ae58dce168c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776424634.96/warc/9122c6168167435da1381505e9b70dae723232e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776425157.62/warc/d460621bfb71424538d8727c836ab34f2f939b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776425626.86/warc/2349df8535fcde67e183a5d2324c3a67ed85d653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776425666.11/warc/247e27757ee44897fae79ebd50fad34dcde8817e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776426171.91/warc/a4bd065deb5d34eeb754ba1f7c74889355a37462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776426419.6/warc/729aeec51b7e286100d0091bd62f2154eee1212b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776426486.74/warc/d357d52ca07e2ace67e2f1eeabed0215d2cebf8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776426734.39/warc/004cd7d4109a8f72851b221ef008ec4ae6346646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776426922.10/warc/de3e4467fd81b309d112b8e5e5cc4b04dbc8811d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776427226.68/warc/1e10dafd844ce99fe1bc0d2753bbc5886b1276cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776427481.71/warc/54d90efcd97679a81f61f1383f59913d18d3ceda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776427615.55/warc/4b1ee429579f6df29bceb76b6dab828440ab9dfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776428273.44/warc/8514f784370efa0244d203e16da1463f49433075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776428349.3/warc/ae1197968f5b5ec2f67c903139e504f8f7ee4af9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776428735.82/warc/b3fa9834a5539f6b42eb12bd3fa55434bb101760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776428772.70/warc/d19f92c27438afdb5b13c85baf20f649c281bdb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776429391.31/warc/68528743acef91e00e2944522e83993592a77cbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776429773.19/warc/4634a03db2f5fb0d1eaa6438d937d0b387145081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776429991.81/warc/2a2e704c3d14a842f23d1503f00039f5f9d99ea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776430044.46/warc/c8d527db2a5773a63224eaf680c4cca6354344f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776430107.88/warc/0d5863e54e6721fd4973303a82e0bb3c02635805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776430645.99/warc/bb5fc2d2c63b7d1dea7d57d214501f919b2baf8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776431056.5/warc/b642151331c834558dea19187f9b9d1023508bbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776431152.42/warc/17c954369f7692049fc24333114a6bd4c424853d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776431372.37/warc/ed29788b5d40bad9929e6b1bf0ae09731c14bde9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432195.33/warc/38c1a4e80c0230a2cc898459f0473b4a3f083640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432786.7/warc/786010091366e8ebb98da4c9bb9502adca546e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432860.32/warc/f112216ce8a29d5508498b52f6d3e276e915bbbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432874.14/warc/22d3a51dbe51e9d7fe6c1746f9556e859b4adf67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432893.52/warc/2a896652344dd8006d2853aa6e72a4fa8ca002be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432978.12/warc/bef866c9f3539f663aa9a084b473ce17ab6c1f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776433958.15/warc/17aa7b9c6a9b82fc0fd68eda3ae62e745fba2840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776434088.75/warc/b957cc6588e9a99540a63eb47807cd0436a61f01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776434099.83/warc/25e67546c84c55b135f8baeeec92adf096cf8b8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776434179.26/warc/4b7b9f0272d3262a56f0ffa02cf0017410cff49c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776434475.94/warc/1d48ad160531ed0ed5d455d0fa02c1f106ab7c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435102.41/warc/d53116f723b9ce02031d37c29474316dab8f20eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435439.76/warc/d882e2abf8ee5e231a4ca7bd6c694e0eca07856c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435465.20/warc/f39884c5d91e6800bb4eef3c7ceb4cd8959b9cac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435471.2/warc/20418ae8193a5d719bdf6fb748e111d2d761bfbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435808.92/warc/39703318cea2ec04f4d00fdd1d411399878c442e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435811.28/warc/bc013f957a64de5f549035d74f97e1a88164c8c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435842.8/warc/bf925b412a4b0ae9ca573cd3b52e1e5f54339290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435941.77/warc/24e046f20cc8fec3ace73d2b25cd71d143ee6733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776436274.65/warc/a11552e574c60de55b88dbce6d4c498eef925f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776437232.85/warc/0b38ce427d58397cdb65ff3122f8618d9c9c8f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776437410.51/warc/4722b1e9a27ddd2be7f706f3055974951437ea43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776437493.50/warc/0c031f36e51656bc2b82c8283902882cef366fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776437601.53/warc/8169e845b81c9159c2d19ebafd086c599a0937be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776437611.34/warc/8ffef62ec7e9147512a08abbd198f0cd4f20cf64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438008.40/warc/5c6d6a857726f6232014b95fcf4bc995a221e7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438278.73/warc/2eee814c3838eea2c98924c5fe3237d13a7e2a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438296.25/warc/b456e27b6bb18aa9e728684e8580e5dd6c3d42e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438333.54/warc/1edd4433977a8d7fdecc6cab0a6e9f9f839655ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438382.45/warc/93e0b4de5fe0837c9e115726ea95d608a0068137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438441.49/warc/fb3ccfa8ec9d9b1ae773c1a0931cd643104e9cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438539.21/warc/31293f72c94495f2cf29d688f387a9e7459134a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438683.27/warc/1adff1e9f2ebfa475803a386ac7f9ac0e3e6ed6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438940.80/warc/7599b38d66772bb2b3108364d2a5c56614ccfaeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439014.63/warc/71497503904b840fd5885688a325aa2885b0136f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439293.30/warc/1ea20e140f75dedaca25a61db8d5b17ea71021d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439565.18/warc/d3da87085fb256dfcd79f07213f8cd83dbf03383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439852.43/warc/e8ff27e295a460204704b5ef43c4f78e6708042c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439916.87/warc/eb8777af338d4907edd43b2b84e152facdf147fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439950.90/warc/16342d173e585784cef5e3f186b0dacdf9c16868.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440024.38/warc/b1e5dad24873dbc2b25f9df26c123b34eed460f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440026.93/warc/b973493b5ee31b98b21d71eb2c9feb7d738e9131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440175.84/warc/aed1cb73320e29b7089a59ef90ec7ae36c3d9f94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440207.64/warc/5aa0d370eebebf7b67a22ea20fe3bfed68a7f1cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440271.67/warc/74996bc8009da27f19b2e11bb5c2a00c65272bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440364.17/warc/a4a61c7bb3e1a7d1a7aedcec5e1670d9d46a4a73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440593.58/warc/4a35bc839d184fe801effc2b4a214f108b817237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440762.97/warc/161c825570ef86218b94dd1ef1079251b1e46263.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776441023.78/warc/3756bba2007fe362d68324cdb00b4302dfc1ff52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776442757.56/warc/2a3315817b7e4a7cdb84050eda254ac7d0dd489c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997857710.17/warc/2804550f0b762eb3388eda1d68fd3a1aed21eb3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997857710.17/warc/62e9fb73860938728c35be1bac6a8fee70632e48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997857714.64/warc/9c950202a865e0c7d6754ad3ad8212f06ca7e4f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997857714.64/warc/ed16e3867113ccff3346a67d6fa527139b7c1fde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997858581.26/warc/353876f42a23ff14f177112b469fd9f0600b4ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997858892.28/warc/37451e004f139f86bcb411ecd6915454488a8ae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997858892.28/warc/f92e44294a1e366425e2681bc935383a072e38c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997858962.69/warc/4ab050f42f8b2eb188c4a60f3d23feec58317b8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997859240.8/warc/36cac32a60bd71b18ec17305304a4ab44bde2d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997859240.8/warc/646729f9ef8da88b2d7cd3e350dc2a969df779a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997860453.15/warc/22f97ef29ad0f06fc97484b74ffba1043b800692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997862121.60/warc/3e617318b07a4d133248e0372cd20d391389adaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997862121.60/warc/67205efde8194512b4199ab35cab186d8bede4a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997862553.92/warc/8eeda543b19a634a60f6a9512f8e36b272a7b897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997862711.66/warc/5d2f23813371fbd3981dcc7b22f27d2afacef496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997862711.66/warc/cfd68922a38968a3260c2ac9dc8fae35bb387e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997865523.12/warc/1224f34e3899aae3134fe2c71c2c7e617e613aaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997869720.10/warc/6ea48a59d1e27598064d6ebbfbbb308d82aca5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997869720.10/warc/d342163f7fb457b30e8f77bc3adb886a6be886da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997869778.45/warc/2fe13d0a638906d65e7f90861329fc8c748bc3a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997869884.75/warc/22868c6ed823b209f9c013a855b3bd8326c5ebc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997869884.75/warc/d9af4d4e24ea057db1546adaf0c9917de33a7470.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997872002.22/warc/391c7f7bd56019ad51adc00e0040d990fa5526f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997872261.34/warc/22e03c13f1423f552d6532a0503f0e3c1a226123.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997872261.34/warc/ea735ce748efef915b830f25e8ae54c4cc8f85bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997873839.53/warc/2f67379293cc8302e89f82a7a8bd69f132d0825b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997874283.19/warc/3f93356fe0ffecb9f0e692bcbcdff7fe184e96e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997874283.19/warc/ba09147b7566241568e9d0022245962b490ebaa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997876165.43/warc/9de1c37975063ecb8b9011facbff9f8fcda4938a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877306.87/warc/59849dc7faf5c2479a4d78ca9434b3a182529bfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877306.87/warc/b55d7d2ea5663d382efe78c49b100297173673cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877644.62/warc/5c5dfaa194c8888096dbb3a79f398bd837eec0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877670.44/warc/345bccd98eff9e1c8bd92e723d46aaf0bb55ed2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877670.44/warc/63b91c3c86af6f3f43572f349126fac8984fa4f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877693.48/warc/dd75acc6f95dbe7d3a6ba1dca01cd125f91bbb4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877869.73/warc/1ae71c21b971838668487f6614f5670f44935644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877869.73/warc/801666f81be13e2a1da808b3d9379c1f572a7ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877881.80/warc/95503a57c2034232d6c9aad6a34a2657de884118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877881.80/warc/f5b5eaa1cb0198b55170c93600a82033f4557171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997878430.52/warc/adcb184006b70f0d6eef6287dee5c15c6478cbc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997878430.52/warc/e59d3fb77fc56addedfe37caca440dceab247b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997878518.58/warc/7b0fc0c82d3a3af37f56aa0de79499c939a18c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997878518.58/warc/a887f5f9b548fecbebc22c0d88650402aef6eef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997879037.61/warc/bd1860f7c6859d33e781f357e9f14089b7df7d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997880800.37/warc/1e9aab6a65f60002bc3a52f8d7cdf75f51b470d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997880800.37/warc/901438b86d77879bf65fb82168a1260d6aa9da15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997881423.31/warc/917a92b3dd51c300e41c626ef983089258513afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997882928.29/warc/9b60bd2a8d2202d27d9e8d06a3d333cef81f4609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997882928.29/warc/9c43ada7f8475f7b695dcf8591ac1c855b3db36d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883425.83/warc/90ba40b9b35af5af1cd4e90e1592c940e7d61acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883466.67/warc/6ac28fb5e1372243f1e406ed469ae188bffd8861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883466.67/warc/d64637b6e92f4c5f8952bbcf1278dbc36ddb2b71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883468.51/warc/c9f9b4a1e52c6b25b860eaff2181e6652de46ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883858.16/warc/4a8ffb05d9e3a2a6f64cd42d5c19bef26a1efda0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883858.16/warc/ad77600a2cf770eb320ffc402c40e0927046e396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883898.33/warc/2b9294182b2e473955d84825d1d504b7d6a220ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883905.99/warc/07612a05b37b2f958e595188d3950ce399b36dcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883905.99/warc/b86578baa3932c19b554a677aa328106aac84660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997884573.18/warc/50c3fdc6278debc4b337df74d1ec5d581954e6a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997884795.6/warc/b50e218d999de4bb629b7f72869b70b707d53fbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997884795.6/warc/bed3a5038c942feb22f197496da432d403362f1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997884827.82/warc/ed5438142c229297c64c47f1ccc23b1318e576bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997885796.93/warc/6958423c2cb82f997e10a0752988bbdb5bf2f8ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997885796.93/warc/f34f8473aad7e11d92745626ec7bd5830687b262.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997886087.7/warc/46be1cb6236ff82fa5f1f794b02913f17066177e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997887829.56/warc/2376ef8b25382c6208b0e460cfd2e8e305db1ff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997887829.56/warc/dc74905eb0488fa2ee4b5551d028b798190a0735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888210.96/warc/97ec479a5a67df6268dea348312e95d6aef81226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888216.78/warc/3898bff88c8d5edee95449a48a907983a3686dba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888216.78/warc/731584de81a2fa4f306b65ee22436d8d9fb44358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888236.74/warc/d80682453a3e50d464e6439e9b725d07957be2ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888283.14/warc/9603a8f99418117a7303692b21333bad985dfeb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888283.14/warc/b33896f51db875cbb5437930f7b80ec3194f79da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888303.30/warc/3132f9e377cbd9dbb096b6336b1eced85134d33d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888866.9/warc/60c1565a390f6273b74b62a619ca7667068d09a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888866.9/warc/af09e21f56b4add4d957b3eb77bf3a10fb10868f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888972.38/warc/3fd581a5820d268b32f641dcd8e2fa59591865a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888972.38/warc/97eb2a912baf0ca5c56ae9c97790e376c973245a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889001.72/warc/2717cbc78344007c9afdfd8614341f1b038cc1a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889001.72/warc/689acfcb880d15d6d617db98860a50d24be4b5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889106.77/warc/5b5f3333f418fd9b026062b43f1a2b94c927fdb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889106.77/warc/77cc47e492bbe59c3c7237d45191b9d62102ed98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889255.54/warc/29bbba640aeade9aab841997f1514022d68bd85a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889314.41/warc/055e4180bfb72188d47bce25c76d15fc9291763d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889314.41/warc/9c826fe119ee32eed7bb40310bf8a2f1b5089b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889379.27/warc/db8102a919b301632116e29283f3cd1ba40872da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889455.65/warc/e0fd68c5d93fc2277e475ed2714310a5e7ea3986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889455.65/warc/f1b5e3a43cec42b7a112939fa564767cab5804a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889622.85/warc/1c968a6cb4a5bf6fac7ebe650c65df0a9acf0f8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890181.84/warc/3795d87cf0a4ea0380330fc3a59a6dc4d85c12ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890181.84/warc/b1d0227c5ecafeece1b4628ea6320b9a4ffd40e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890199.39/warc/72482e4d478c8fd59a4433535f43e97b0687aaa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890211.21/warc/c030affaf09c5eb6df10a01dbadb305901e93594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890211.21/warc/fa0e425e51698ba5e5fdd3efbf789032b684adeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890773.86/warc/5eb3ca7fb277c6aae4678af5f848b6bf445efb68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997891176.68/warc/0326731cb01f41796a7230ce6f43bff00f698f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997891176.68/warc/f2a879b49c1bc330b33ec8e20e8424863c1dbe9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997891953.98/warc/e4f08d070794764c053918d7ab3a4de544395f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892084.13/warc/461fa4ef3ea2d71e041543ab19629b28e9f5ab9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892084.13/warc/cab271f7563d10f71204bfbe7f3e9841b1ed35a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892495.1/warc/bec5befb8278d11c7dc790cfde6f73480138546e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892557.70/warc/254f7fe77af5fa5e05dc36566633cbb2e37c05c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892557.70/warc/25c918c5b8c15517a712d04c69a40c7a2d9215db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892641.2/warc/6d80d751dfbec8646f514410b5cf947a213e4fc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892648.47/warc/bdaab8cda06797e379923a568ae1fcb4f4a2f40f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892648.47/warc/c65b8e7a8bc745f5bd3b38fd7561cd288bd108b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892806.35/warc/42f232e7d1a33df696b1b160248611746842d9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997893859.88/warc/b1e3fb0bf0663f492c1f71f5d2cfeed1bb893b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997893859.88/warc/f07a8cc92771cb743c69799d2eac7cd53316374d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997893881.91/warc/105a84e8bd2b68bf90f19f7c60d026745c0869bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997893907.95/warc/0e4e3529cb7533ea1996de4bfb8594a6b0b5ccb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997893907.95/warc/fc0f6a7ba7e39d37d47cd635705691c073add86c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894140.11/warc/9da681524e8318a723fe637da9656b870e43a5e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894151.32/warc/02d32be77fd6ded6879e8e7948906c6bfa35fc37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894151.32/warc/cd44b422e7e9ac6aca24f0e7ce83b9faf3d57854.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894250.40/warc/3514810992fa64fbdbe7e901136f14066465600b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894250.40/warc/f547f89c1df11af73f08825adbf643c521cbf252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894260.71/warc/72a5a21ccfccc9b030479a496b018e556d99c0c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894260.71/warc/9fd137f20b9dd38d42d62d0f8b62497bf5e0929f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894275.63/warc/27c7ff83c2334394e1a397dd4afbfb102d6e5bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894275.63/warc/4ec7a5de855a681357e83afaccbd78b9070169cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894289.49/warc/d5aafb9d3b53bee62250af3da9e4bc8011e12a9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894319.36/warc/456ab53b545cae350ac4105ac8ea8c7c0989bd8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894319.36/warc/785ece62305bda744d6b1be2c0c778606f409097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894378.97/warc/b411b4d1c73020daf679873754b83c03b7276a81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894473.81/warc/914151d1469c421f7b44ad9f1b09368de362ec34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894473.81/warc/ab4ca4c934b957f7ebda9cc5718dfd5c2d65cb94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894689.94/warc/9ec915d5768d7aa1a696071a3f37a6337ab0dc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894782.90/warc/2b529da7114dfc73ca462605aec1c58df8d102da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894782.90/warc/7e508545c4c4fe2d822f2baf82e639beb42bbeab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894799.55/warc/702817c776dc70910049640d92f8e7ecaf7cbcd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894865.50/warc/6baa335ce62eeda39d2217079c9705c19e2ba6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894865.50/warc/e2a50ad653f68f21e7f615dc2d223c1f3b74938d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894931.59/warc/da6d5349a7404e75dbbdf34360021ce23200dfd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894976.0/warc/8864dda1012e222f4f5876080b56cce8166cadd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894976.0/warc/f6b47253753120b045569722da3516e6629d054c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894983.24/warc/d28b2d153e4502312935f623edd9acd18332b207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997895170.20/warc/177dcfd03d8cf761a5778f6c54ef778e161fd031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997895170.20/warc/fb83d6f7aa4189d3dda9bab542b41c8c75a9cfa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997900573.25/warc/6c623fe8aac406f2b85a29a19ab7a2453ab3501d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997901076.42/warc/0157a8c82902e8d616bf9f6bbffd895dc0fd67f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997901076.42/warc/7fa4dde988210c6faff4d6b568d703a533844a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997901589.57/warc/ddda94ef605a985899fda98ee418c5d907b5c56b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997902579.5/warc/426c8c402253420048fc212dd5519aadb301346e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997902579.5/warc/5de0b9b7a26b3b4b5cc96e4a4947e78cc06e209f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997903265.4/warc/6e14bf486fe75b5015ce3ca8e6a218127981d416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997904391.23/warc/4a371d97b86ac67cbf2f1322c512bda82803a983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997904391.23/warc/6bfd95dc4cf2adf3de0a46d481b4efe94eee3770.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256737.1/warc/5a46b830127b97bfc1b4257a52898d7f3c8dc3e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256737.1/warc/69fa064de11e85d751152ce1322e899734796086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256737.1/warc/718e0eafbdead2a89dfdce3f2a938bc75dc55e41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256737.1/warc/cb91f73c836e94cba9090e99b7bc0171c85ea84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256757.9/warc/3a78607dacef2db4ffe5378586c1ba5941e9e40b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256757.9/warc/5a62b2f281ee2f3bca22e75911773c84d11863c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256757.9/warc/a3006bf380699daddc0fa589b398e01447e9c15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256757.9/warc/ccfb494276890eb4d87dd473ab78ab36134412a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510257966.18/warc/0af8b3c8450499a0e119353b67b357f1fc8ae54a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510257966.18/warc/320e1caae0d9ea426ca5dfb6b48ffd64990c93cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510257966.18/warc/3ccb006fd6141a6d554b79700454994c019be322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510257966.18/warc/c490fc6435419097bf0fc8035e0debafedff64f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510258086.28/warc/1cb05789da88b00a9c2c393a911a4fa2f73689a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510258086.28/warc/2dbfc6fabebd69b15dc9d728d56009ea300b5633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510258086.28/warc/64c032d0814b0470fbe5163090e73f062a0ad379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510258086.28/warc/876be4083b6b8f91c7d861715c28bf5532a2c926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510259834.25/warc/691d1f8a92c345be96dfa91d3baeaceff6d04b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510259834.25/warc/a6e5b86ffa7397f545eb0017a304bc4ee4944598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510259834.25/warc/c7aae23e54b631726f7cdfdfc62f9b614ba7e53c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510259834.25/warc/ed2c15e298182a6ae8b5707be3c2e48ea1856fb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510260734.19/warc/25a476197268e91a03475533c1d75ea63b631392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510260734.19/warc/4567ce16fdc61af50160ca7a0aa9ef446c109bf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510260734.19/warc/47a461d0b24c8806d08ef9ad6eb21b78df689b46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510260734.19/warc/63ee3a99910234d6ac3937c9bc78726c99fadfac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261249.37/warc/be73268ba192419295e9c8fa84e25cd06e7ede5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261249.37/warc/c0561a64dc64cbd70d3b2c82c75c32ce9a60f1d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261249.37/warc/ca8e90246067d9dba94b6c9bab05e535decf5ceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261249.37/warc/f61ecfb157365cd305be2900e6db93315d8e85dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261771.50/warc/705fd4ceb1d53c3f57889a93a51dcc1f2be18379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261771.50/warc/91e6f6584e3a2bd7198e54febb61e1b3c6e01fb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261771.50/warc/c159239d96ce3eed4548c89b0ab69b33399d36b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261771.50/warc/f56cda61c3523572d853a8d0f4a6cc0633d690a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261958.8/warc/17b02935d2d89892e33991ac1307616faedca592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261958.8/warc/55be1901cb6c1937451bbb594bcd47fdadce3f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261958.8/warc/67910e2123e1c81698a6a56cc7170e8d5527b27a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261958.8/warc/ac7ce22c119c203d2288f3654dc25810c5e0b1ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510263423.17/warc/3e0fcdc85192c7bcec00ac312d22cb1f582a33e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510263423.17/warc/8d302bb249adf7e843f2ff06be3a9f0eeb2f1508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510263423.17/warc/c48983df42976134b5c763c78d8f35c5230d158b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510263423.17/warc/e5ed9d7b75114cac43585a3e71d3b082abbc49c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264270.11/warc/6131f83585d3473e9b19731c0a4967f52f644af1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264270.11/warc/b7714c3bfe3a4acc261f4ae13d8958f4c1d4c0fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264270.11/warc/dbcb412ed564f7e06896fa152e437d6b2f05461e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264270.11/warc/ed506a2bcc74f7493b83c9ad0cb61c986a84a370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264575.30/warc/64514344d50156fa8d58c8bfd586bc840dbbf666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264575.30/warc/68f303429be57b46257a2b67f406943f068ad5d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264575.30/warc/88a4d59703ffa3f0adb57e9f6a8bc4fc0f2e99a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264575.30/warc/dc960cbec9a4f18fdd02903230822956bcecabc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510265454.51/warc/30e8cff926f0bc8d15a3290f976736c7a874a813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510265454.51/warc/88d5e8d0468a89cd49b6496a32fdbf5b098b549b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510265454.51/warc/c45bc9e1454259cdb67a4324dba43d4d1a2ca5c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510265454.51/warc/cb18e38284ffac3790d9fed4bf8ffc441891080e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266597.23/warc/56f620901542c132f2a915b4e28799323e536cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266597.23/warc/61e9f6a61ad4f9007825e651837d7cf089ef4ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266597.23/warc/6cfeea97eadf5e7114cc48602f8859512a6d4eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266894.52/warc/2743b5a142e6e8c9925b7281c239c0a1a5345c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266894.52/warc/41a03275d11d74abb22165731cf94d22bc5c9327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266894.52/warc/4b380f008d29c50efb0666f10ac0a0d79031bc0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266894.52/warc/810c1f5df66b619932183708ba16e07d5c96600f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267075.55/warc/09b5d3f419450bd13e4ee7f3f15d6c08b0812aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267075.55/warc/531f5cff3d8bd8e046fd79e1c0d4f75cfaf15e18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267075.55/warc/bd6f75ff994f71e158ee6d47994d0cff4ef1bfa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267075.55/warc/ebd2fad203e806d923bb8bf29c6e4cf000443197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267330.29/warc/5fa163fb5f96d4ce7d1cfbd374070fc9b44148b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267330.29/warc/955893169ba750b4b4926f763084d878180d7c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267330.29/warc/c60e6f216552e9a1a9875ca455be4b973e8b8019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267330.29/warc/da467df6b19af0c22a4c1082834b58038770dabf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267729.10/warc/22fd45d16e060a5923df9f16845fc780a9894ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267729.10/warc/574aae653d9587b861e9d088479388a3af53e6ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267729.10/warc/8b2ccc90c88879bb66ea0d674113493fd9dd7689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267729.10/warc/e05e4ed42c65e2a4e04df8cf459c06393915d39a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267745.6/warc/4749189e0bf06b174efd94d6c97a6c8582b4db56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267745.6/warc/4bfc4c9be9e106d494e0a67d7624a508fd818bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267745.6/warc/ed80470271109857bf9028ca4b0e0f695bbe1a5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267745.6/warc/f99cc22abf757e38a8f79a87119052396c598316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267824.47/warc/35e3021d390a3db46a8e944880da5bc28675cfa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267824.47/warc/56781e04da953638c4d4bb067e39f582f1adc3e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267824.47/warc/8f2aba177d34ba4d8add750604a1974ad1a635bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267824.47/warc/9e82961221a984f5cd131419bcbdd9f2fb7f2876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267865.20/warc/2dccef76df36fe5fe6646db5ecc6a7e09d9196c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267865.20/warc/5313e67d41b56cd3a7d73580580c6f11f1ea433b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267865.20/warc/e6099535d31c2ad067bf9db7e327bea45e0a84d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267865.20/warc/fe7d42c437046f88129f469342f4c440e30443ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267876.49/warc/27c5366d6557504ee1d6b3d8963697a2dcf805a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267876.49/warc/5597b1b2c315def3345008ca4545e71f30c626d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267876.49/warc/5d5bd1403d0055d54afe747bb65a05d302cdaa2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267876.49/warc/95dbf7094167236f79ac775cded9ab03bd9c8ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268363.15/warc/6fa9fa8a15ea7c1e8d86f3efb7357960fb1132c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268363.15/warc/77855c90d0b7b2091c0d2469114d651bb4e8d48a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268363.15/warc/88b8ec5dc1a9ea4b46cb6bd2c877cac9884ee4cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268363.15/warc/ecb825ee8513431ae7144353ec7e404091d5eb89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268533.31/warc/241e9ba0ce95429c8080fb6606b329b4c1d1158a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268533.31/warc/4a8f80b74f4b594fc3e54dd5cb2ee8af05fe30ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268533.31/warc/c169eeb276c6fcc3a3292845e53ffdcc8299c4b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268533.31/warc/fb63713f95f183990d081275071ab7ccf1fe7625.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268660.14/warc/60c4db5d8d4b3a95f6c27d597a5f27a9361a465d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268660.14/warc/afc71cb777b7b99790f4c84157701cb51921ebe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268660.14/warc/b8ba264f4237f7a1c67f9055a1302bf42c920fe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268660.14/warc/f364fdfdf00cc9b38584814adfa1bc1dba252aed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268734.38/warc/297b25f02bb2987bd2f7091ccbfcfffa9588915d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268734.38/warc/3fe5099dd0cac427063e6f78514cfd8534559f94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268734.38/warc/971b99c1e3aaf7daaa20f2cd00d5197a07fb9ed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268734.38/warc/c98721acc5511a5cfb68fc7982e63ea6623c0cd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270313.12/warc/8d0cc51c559840e4db9ae2d73cce710cab95bd22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270313.12/warc/d42bb635527130b6807dfbe651e634dc0b160deb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270313.12/warc/eb5a7a73604277587e5a5de10928280162bef011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270313.12/warc/fc8094f2c5cd65e52697ec0972846617c2614b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270399.7/warc/0ff2ff4f324c826f098ca4db16277423e3f46b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270399.7/warc/101b1feb140dc38536e3a3aa2eb5645874311c9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270399.7/warc/2dbb731a0084dde9b57d869104eca0ab6ea985e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270399.7/warc/b5d7fdeb91dfd350a0f99f3467a2af6685bd6053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270528.34/warc/02724977313ac39fc96e3f32d9f6baad47775463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270528.34/warc/68b5f796da08eeb1392f97fffdd4ae08c3390757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270528.34/warc/f87441be53b995db186b9c5674938ce5405d0044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270528.34/warc/faf4bd8d79671b78c476d9f1f61591d16e2dc3a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270577.0/warc/0c72c3d4c4d0becd836ec0e91ee71149a0efdc75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270577.0/warc/123c72cbaeee03d4a835a1279824ed058d7cb9d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270577.0/warc/7ef298e68d94a93afd7bf28fc61235cff09add0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270577.0/warc/e104ba0bcc0f590e20c9dfd2e77e5b377f77d447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270877.35/warc/2bbfe2ceb014b52bff6a4c2e3de500416c3efb12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270877.35/warc/6a5570aeba72e35d573c459e4e217bdca549e20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270877.35/warc/d58478efecc4603a01c5109b01d82af9f3a710b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270877.35/warc/f6a336005c05f281d5add753816a6c45919e6c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271648.4/warc/0a1ec3d4b7949e9fcad9017ac506e1229200166f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271648.4/warc/1839dd0360499b9892628f06b6ff8481a1429a59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271648.4/warc/4109e9d5e676cbddcdde5f4a36eacbbf8a09c6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271648.4/warc/74f3141242148a55f9c3ee60b6847815cf2c3fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271654.40/warc/1f6e6f2ebbb6b70fe9977a6e2b4e3ff32f0d1e21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271654.40/warc/9a364ac5ddd87faf416c95f43b5a3b350cbc0c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271654.40/warc/b56a8c79beeb1074bc62b86f208abd5e05bf1f4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271654.40/warc/cf22c9fa23150388ddfd82f61dc7a17685d73128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271862.24/warc/4e072981eb5f8e8a936968a2e862d69fadc47132.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271862.24/warc/5e609f6b964ca5083d5fdceefd656173aee956e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271862.24/warc/c0e053a3266192bbbbffd3c7018a902efa9536e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271862.24/warc/ff21feb28e1c6522dd9e8544e0c03de0612cc3a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272256.16/warc/be035a4adacaa6ff3c4c44c3487543cda893bfa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272256.16/warc/dcb74132c596c9127c149dd62f965cd6ad609ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272256.16/warc/df18be2973250570f8b3008ad5922e580ab30930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272256.16/warc/e9827640e697366ed99289c0e2af6165589aa211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272329.26/warc/470545561661848263b521d36b9e7947f13933b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272329.26/warc/5143b62c175e93c49698b141bb8c24700b910eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272329.26/warc/5eb5ab8a3fdd6f469df9c859f70fdf4e9bbcc160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272329.26/warc/836ce4b6dc70e7172bef12cd869eac2c1cc29ea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272584.13/warc/129e7e61d40567e590b390f6c76481cb686f0d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272584.13/warc/2e3b78df01ca4343ee8798e37b3051474140849d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272584.13/warc/3e38041c82f597479e77716c350892db71e340cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272584.13/warc/e5abe3a1244a204a421c35d353a03a22cb9357f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272680.42/warc/0409448c4cf9b980f088cc73737cc275893e3ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272680.42/warc/51df2e6cb50d592ca1a84029494f3001feed74b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272680.42/warc/5c5ea95238531a1faee4f1fb7c5c4f61ab551bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272680.42/warc/6b08025e770f83286489d14ef26078b0c24de24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272940.33/warc/79c2ce190b08d4e6c8e05e5e7e9c5922e6a949ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272940.33/warc/ccdf2978c7426da9866a6153596375c1751234d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272940.33/warc/cd3c150955608d4c40e3468fed8d26752cf59141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272940.33/warc/e06bc360b44047cb52350f794922d9779a87706a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273012.22/warc/29a6355df34caa52718c9b32e052de2dd3ffb8ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273012.22/warc/5b5f4ce30d42f1ef5a249d1624791523f39d74fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273012.22/warc/718ea462a9f70b33006df30387c1c51291a3038d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273012.22/warc/9ce965f6d0f172798ba043936f428f32c3a83899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273350.41/warc/0b31de798f115b4003f385348c2471d897c5d5bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273350.41/warc/37c779c0ce37811e2a524fb1fabd632cb0a8e1b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273350.41/warc/5adb1ae181d537ac077faeb1e91b9917ee14786d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273350.41/warc/79c3fac84d3819ed185f450b0c888aeb5e543613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273381.44/warc/23d0009d4565734c356ce7b5080c27168ab914b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273381.44/warc/a611f5ddb2d6cdf059e16a3b50faae3167ea9365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273381.44/warc/ca9d871a3d69fad2e7f9a752e7d369c7803d3fdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273381.44/warc/ff2e14eec59273c6912dc878aa7476b905f94eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273513.48/warc/633d59ce6bcaae869de402a0f96003be88cba816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273513.48/warc/bce53b5695635d16f63eca781ad7b3ba9da67b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273513.48/warc/de1995da771d6cc5e0febc5e26c386b68cd7d97b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273513.48/warc/ec439a1bda36094b69e707e0bb19d4209c48f5a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273663.2/warc/7e38dfcd18fd95e3bf5c8f3053d1e5c06a0ab9cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273663.2/warc/98722a1a0a3aeba990f3b4fb4f85bb0312da2fb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273663.2/warc/b23a85cfd2cef634b0c1188c82702f1627bd46eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273663.2/warc/c47e02f676b5720ef029e025b5319c2ca03a5bd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273676.54/warc/5ed338cc55af455bef4bbe5c5eae59d77b862f92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273676.54/warc/c23fd8fe890019da59cbc998fccd139d038d8d80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273676.54/warc/f0e8f78044cfce13c948fe1ba33fba951d186c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273676.54/warc/f4c28fd89ce118b0d7a657fd8c414b146be05b72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273766.32/warc/4f33dfba0daba8437abfe2dbf80753123bd66522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273766.32/warc/727e3770fc1720734e5f2baa0a7fc7c82db1a688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273766.32/warc/9fb81c125118a8ec90638a78f250eb64369dea06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273766.32/warc/a01c00145379af63f40be24ecd3d9a58cea29b08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273874.36/warc/684da086545ddd3313cc76c3c3aebcecda0c91ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273874.36/warc/8437fcc13e7eba465f51e53ed64f26d1010de488.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273874.36/warc/ad0f1667330bb728c5b6453b28a37837d684bf7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274289.5/warc/4038a1c1c1e315fcaa2917ecf5a453acef3e08be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274289.5/warc/6bedfd87bcc694496868daf5863e22017b26360e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274289.5/warc/82bfeb44bb3079dde2350511dcaba70ebd549825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274289.5/warc/e57c3e2136b859e64ff0f8aafcbed3fe74de1294.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274581.53/warc/569169617a296e1da5cf989766cfc04bba55c844.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274581.53/warc/6503e4b3be45006c46b78b04024b7434ede83eda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274581.53/warc/6935e463420d036b2ca54cc3c3fd901072defb80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274581.53/warc/d12356dc52eac82a090777c47fb86dbcf9f47ff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274866.27/warc/3a30dd6a0f631316c5f55ba3c9d3cec3e92d38a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274866.27/warc/6f571fdcd170990cbfc3ed188ea337f56785babb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274866.27/warc/8d0359716044a5b7e1a9e5b2a2a70a7571e9a461.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274866.27/warc/8fb310ad01f355f96c058591a216c604258fdf92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274967.3/warc/0410e83bbcf66eb3ffa50a4d84a884697a83ffe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274967.3/warc/76f6c6208d87879ee99a9d1ee2d69dcd9704b308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274967.3/warc/ae6d2712e1538516950c872b154fc8e9d5c2b68c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274967.3/warc/c3bdecf5a82f53450e1e87ee71730665f156d4d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274979.56/warc/050b61a5df405c2681be19f00988afd5ed7050d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274979.56/warc/3cd3771a1ac98406bed1ac74e7161e729cbc5649.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274979.56/warc/51747ca121301178bd3e93ded44c2879dbe987d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274979.56/warc/9c734fe7c3261805670282e6925b083b9d3b3730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274987.43/warc/04f1e7e5675f560dad4faae90870e314590060d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274987.43/warc/0f67a24ccded071efad008369136871249229dfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274987.43/warc/2a0b6da6c34cafff5d5bd64bca4552234b836037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274987.43/warc/bb93edcb93db2ff39ec1b4e5b19ba3e290076b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275111.45/warc/06ad36f8c654d5b2b615c5e69da3e52fe82d998e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275111.45/warc/1fcae5b9b258ea3df3d8d648ee8b444d07544b81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275111.45/warc/48c2df597f273f441d6aa6c331121c32178e5c64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275111.45/warc/962323c27e35828dfebbd773c032c92c60554ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275393.46/warc/445a2e17225de74be59f1d84bd28cb2a9609fa7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275393.46/warc/585af027f7bc886ae81a0fd6ab3dc9e8c2aa1820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275393.46/warc/7711702185a5aafcfe4405a6f61850f7f94a47ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275393.46/warc/ec6433f1fc13373687c222925a36e5cd1a342e1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275463.39/warc/68c9a7d1626fedabd51bb678c66ded0c4b482338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275463.39/warc/8e7a3ab5e9ced450120e51cde90bb720c0a9f3d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275463.39/warc/be74e6344d8ee3cbfc4f6f5c6c9fed2bbcbafa35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275463.39/warc/e1a1a8426428831ecc309d3c9d45b5c110055faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276250.57/warc/3d589e9979a11a690806bff4323ac6d640c5ae27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276250.57/warc/790fae63df4afe386a0c2e3c7951d537a3033d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276250.57/warc/7a5711927de28aa075ab1a44499ae286ce05edf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276250.57/warc/bf7d05ae6e537812773bcf3eff90b20b2f2b957c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276353.59/warc/10468cfe6aa40262f0686c25cf98153286347bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276353.59/warc/67b03f96c59413650faea78d70170e7f5e7bb1d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276353.59/warc/b5945fa6bcf1cef2d322c7f780f6f32154e5da42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276353.59/warc/c00f51d695e91f2a89ce0bd34e16067ec3b85f6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276584.58/warc/05cd2655deb86ba52884ce31ede7f1ab48e9c039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276584.58/warc/3d3fc4e0c818ddf787eb4419bbd3bfc0b268ba80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276584.58/warc/66b32d4b25680055503feb66f0373654a871a584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276584.58/warc/918fccfc8de819df9c5c91f84aca03211ae36955.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510280868.21/warc/22a2f2cd9222e04c4716147613a0c5de9d0dfabd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510280868.21/warc/80be330dd7686ac1e27ec234fd4ee1d968ec3f1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510280868.21/warc/936d112ec92506c53b5a37c585a9a6910dacf56d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510280868.21/warc/9bd6579bb24e09fcaf4c9f11f20c456d0cfc36f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510280868.21/warc/a40b9baa3636517cd6c3108c329daa10178174b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/14d228dd214aaaf0bf9e142ebcb64dab5d19186d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/14f4332c9aa1d3b1e8e79ee00b8c1b1c9750bdab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/16e3a4ea7cc89b5a0ff880f0855aac3430304c29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/172f491bf15c2ba0234d353fda5ebaf1effaebf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1a599ad5c4494be2e1a9f935aaa8423dc50ce070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1fb4fbcf87a149061ee871e3ebc3dc2b10bc041e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2031f830da1ce87b99aee386e2cfd8b47665cea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2306d30ac7190eb52ba9e51c92fba03a20bc2cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2345366aa28592e03db04db7b44bc9be75965db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/240e892a8534a88a6f10ec9a8b205b523f3d6b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/275366d346f565cb68742d4a012682b0c6fdbe04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2aebd15f9d166e5cb9157ccf8ec30f7eb4c20684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2ba18b6a73dcaa4927dcacf75c5b946b3e49d299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2eab680c6f881bb54cc4f890129eedaa35d03090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/30efba5c4bc044b54b14cecfbd8e52c42550b3b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3148a2a31e41c697b47c99bb6292677f06310094.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3161ef51418fc84ece373a8e6a469db54c9542da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/31d9c0eb80117a385fe7f7a0f9f9aa51f2a364fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3346f93c9995ca1b331528b1a36d099447069eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3376d9f93fe49cccdf5519f6652e487c9b87a18a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/36949eed52da9b3e922caf55559d4d6507a169a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/38682633b61f23012c5d9347c5cd89550550bfe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/38bb02a3ffc8cba6ff248d4114546c92d94eeba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/38be3516cd29106aabeab878aeff32e250507f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3bb6ea448c7e6c2b7024682c4bfb89ad2d0344ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3d0dfda182b551d3952e82f4427020656a033ca6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3ee7451a1477f15604355413caa71fcca8c640b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/42d2d71ddadb585fa14a7d3c46d4f6468fe3eed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/44ac6876186a014a42c157b107eddfa397345126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/453f383ba90354ebd7a31eef7fa0595889e5464b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4664854fbfa978bb8517807396d5ff282f0c8067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/46db31a9d6ca31b66c87599370966a3b34ad7bdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/46f872c56e9855d6a9ef3e4dd5241de28eccce63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4860c1abcc382f01dce2aaab456ab73679b2e5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4a3abdd1caceee46289c24d4a5a8a655c623010f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4b742ebd335ba31fb389973055812763aa8dcc1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4ba41c5cf1a06a048c7c81ddf1e895c6e38b69c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4eb308af6add6053609e8085e564346350973bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/52048cc8436d1e432778bc86242825e514d1aff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/553d861699afa707be9b66df9668c25ae023c03d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/559f6761fddd5834260e6d8014089e3fa68e9966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/58e20ea4a5f820b2dd1ba21452a4ceebf8a9df95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/594ccc04e6e47519e9a2595d70e58c4fbea8b082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/5eae666cb62c3ae662da5fcfce0259f5042244d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/5f4fe46442af734bde44603717c47c31b72902d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/61a482df0f7a034b9fdca02a31080e4b1473932a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/62eb7885bf25163c4de1c3b60f7bcbc46f891e88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/63e603c1fa583f46cbda1fc20c6102aec27e249f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/64700521a70e768828ec736f27a7db7fd060ea30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/688082d7bba89a517766289e82df7e6db8102460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/6a5a1914d4ee75c637c7660f5fe6b785ec54af6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/6cda89511fa6b67619454a28db861a5ca64719b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/6e93e16c8e6d83362669b3f8be30508852cd5910.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/6f2bbf24ac6d36947d18548fb1507b0165967f6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/6fb747b0277921a8163b7c0bcdd29317f66debfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/71fea35239da7da29510980c64562409e9ad2e0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/73413d85fa777137edcc080d666d123f6be89a43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/7614cbffa25558f19c2d4f87cda7cb9ff63a451c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/783a9d22658c580f93d0078293042ddbf1a26413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/7a4ff9103413fe4cec2372f49ae4798c9af57412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/7cf4b8a76cc53821403fa36c3f861630a1b3eaaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/7f881ff64ad43457b5300194df812474d11c97d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/80b50cde8fdb307e66e4d6c45dc3e0413faf9bb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/80f9183e750ee482dac5a5d14ef448be2c965e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/819bf264fbdd9e242162f245738cb4a9bcde10ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/838a78453ce9a2189fd4ccd29f4e42b368d0bceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/87eb6c95e7752122e4991741b43e7d406ea5b075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/894857fe8e997765ea6db77a210c4d03fd075f9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/8c8bc7e75f3b8db16a295b02ffca81dda51bbe17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/8d263f0e87ba1135360bd2438a6a56a9b2226006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/8d5755474808e1219fc64d62456683452797326b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/940d0b1b9b3ba16e9f1c9997ac6324136b744694.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/96b6c3875a918fbf18c3d9df7a851a0821cb185e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/991c54fe9181bc7c299bf6678c86ee5c94e4744d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/99d355d894a041268ae86d9f13f675aef2c730f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9ac820ecfb36ec74c303cb0d43e5b177460cdd85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9cbcc42550a85045627a339866768c598313e706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9d6b80204303b1d7f2bd00ee0dcbf3c0f76bbfbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9d71c405076813b80c7eab50d5787652515bbc8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9dfded6f15791339ef8fcb998a75bb86397a7b7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9ed3f0df5f56132e4fc767ac5f123eaf8243c3c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a0757590acc322eb406e8865073749f5c95ce263.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a165948af24783cf806fe79feeb8ea559af9d867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a1acea4018334509e14e8fb5f96006870ce08b84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a45d21a187dac357c63f15ef924e87e98f290d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a4a513407f9cbcb592fde4d30644cf7dc38252c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a6bba85dc3323b748e241d551e439ff715eff33a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a735fdc4c6b081716a3d4156f98380e22f65d4d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/ab6b4e1f951996e98258b36e1d01d167bb609f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/acb937856394362a8e03a401f3be214f222e2e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/ae2f4791b9dab13fd476e5c010f8d9b166239fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/ae662b6269591f96df5eb53a75e87416fd0c6c94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/af9af0951d1029ec4dd489c940490a6068e1567e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/afd758c4a60d9359830cbbc32a0971fc04772edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/aff8feb79f4e83f887d56a0527698a414948a696.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b0481ed4fbbae37f6afed8e66fd771e3909181df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b06d3b4b9188292cb3ee5a50311dc99878a5b99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b1437428feccc908288723cea520fed4fc91bfab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b22a8b35dcd85c100925d86c5c6a08b407771fa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b3e20908ce63200f96471450cf311a2319e1132a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b3f61050e93fdf751d6fb833f20d40e1c36e43a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b60b4ca94f1840c253f297801e3eee0ae6479d8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b6787597b49c3e492b82e4d2e067b1d2129ee275.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b95d09f223a94e4e00aa4b9c29a7e21bc88649b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/baf748b4ad63e056c3c1d06e722dc30afdc202e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/bd27bcb964791e2845b996505162534aed420eb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c0257a649cd96455036f0c686ff9adb43a6ff14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c10a6b02eff2b984a044820dd6664494acba7db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c19d5d2b092a874d75510fc8709e1707b2d1f268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c1d0038c3da001f0d450d12ccb670e64ffb33d87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c32e8dbe209c9d898aa907a589a19d8c6006ec39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c871bdf5d06f095d157a930e73c4333ac2cf5042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/ca38d60d96e039e8f79a63fb4d017a53f1fff11b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/cda43d985cbfb88a2d2a0d03aabbb668aba086e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/cef7344a61cf97a34fabfd08a4007574f6c1a926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/d05cd61b8a93ab477dc13aa5be6666ff5039d1e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/d3f8f9776721b3aaae55ab842765c673f5b85c15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/d964354fb8ef61604bfa918d15bac0473785dd23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/db9312009775b241cb66fa5d80d800e6f3c4a5e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/dbb0dc34f1029bd028d49791c12b6f3d4a37c716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/de7f58100371fc0e7b6db8a7af93cd9b43e0beb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/df7632c0768f6f721bc3fb37b078cf9c2b966f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e0d9cdc9e645f76731f5f4e29eeaa7da1a7e934a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e1b835abe5212ed3efba9186ffcda33e68ae70ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e221cb04831a95460e079f192cf746823b796937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e246380325642d9888e6eca00e0c8b12c9cfa99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e3b00c1b3cdbb9eb04a92b91fdcea9a3b1b146a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e5487e242cfe8748761ac1591a19a6308e21ed1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e594cedced4c4f6e7aa2ee29198d759a55d93aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e7d8ffbd7cbc703599489c5733b1677d46e56f85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e80457442175437bfdb02affa98f5763c4da1c99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/ecd429ab35b828a70465754ee773d53c7559ab43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f1a250ba466338403a7f236eeba8ad8d162ed309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f2398dd439e60eda79eb8ccc5908b42c62517593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f3186e23a2909f230c1afcbd5b1ab984c160eab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f40a4c0f93afaf9bce2ff819d1600c9394c5a986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f6a1407dd595e296033c137dd8fbd7a6530f3991.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f860ef545bf5a455ab665233bfebe92a35ccd4be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/fb3a2cf6aa7f9696a005283e7eb3bab3464aaac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/fd6878e9ee9b53623b9a1fe8731d568b61804ca1.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2014-23/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2014-23/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2014-23/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2014-35.yaml b/configs/cccc/dedupe-url/CC-MAIN-2014-35.yaml new file mode 100644 index 00000000..686a93a9 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2014-35.yaml @@ -0,0 +1,550 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/005447e6d776d2d91129a84cdf3eaebc48b5ca17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/02daacefa71dbdb5a2f1311d250b411ca38d723b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/02f6b9ff5b2950fcc36a245ae2c5a859d1c8f6a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0937ed75f9a89fe7eb079d0df538c61d6a1dd6b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0b8935781de9d52511e6d21bfc33d77a6a8f7ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0b9b046d04011c69d604192ba52f07e43b1ee7e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0d316db22836180d5a62db74c27e080dbf8ae23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0db2fd943cbd45a95159260cd23a7d99c24e50f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0e12f7dd88f208da931175b5ea12326551a15ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/119e247160213f7c62f48a8a1213155658da0350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/11ccff69ecef3d6a97d37acdd7216f91f28950a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1284801dce6ef89620c8ba357e65e3d6af564aff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/13ea67b9a70550f4e7a497558a29d86d640bfdca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800168.29/warc/1cb5e5a23b7a5f94a57e9c999851629799d2d933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800168.29/warc/760975be234c372580ef5aae2ddf977f4b95b22b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800168.29/warc/9fd5630f597f79c95c39df03b5d46ff2b092cabb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800168.29/warc/f6e9842e3c22fcf667184f4858f999f51fcd8067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800767.23/warc/15edf65bf4105d262187e8c49877edda92cc4249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800767.23/warc/65c24b0a3599276625c69144f4204304af675d50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800767.23/warc/8be61173483cec038cc298dd7bcf18ef99ef336e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800767.23/warc/b19ef65884854c958dfe5376a36a20f07ff23bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500801235.4/warc/08228ee2d2e27e847e4860ff39d6ad482bab275d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500801235.4/warc/8195e4b529de9e4d5dbf0e2d0b4509396ee2d423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500801235.4/warc/97f9fbd48d2f31df8c121c63ce9ee003e0d9d3aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500801235.4/warc/e6cb4af3cae8b3ed5b4afaa2131da023fe78bc8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500804220.17/warc/b3464f1ab755061cacfedc92c7c92f5ea96cad2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500804220.17/warc/d26f5f63b222aab64af3d94d79bb6104e093e783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500804220.17/warc/ffe6f494878dd54589abb808d39ee5edc78b807c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500808153.1/warc/039ab38660cdf6549cac43c3eed3ac98179de26e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500808153.1/warc/15b99d3edf98880152a57e72174abaedecbbcfda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500808153.1/warc/71677360e334fec463447095b4c55d14a22e077d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500808153.1/warc/b3ad26e3fa18d6bd6660878e2f65a708a2e98dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500809686.31/warc/3ce08aa8af6892b99a1ba9bead013cdff83886de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500809686.31/warc/7ffc64a52a6526ff8a8e2c111bf3107ce776a2a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500809686.31/warc/9a6a19aef25d51cae0a95b24c83660089c927338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500809686.31/warc/eb39e2b2ca4763af642d9d4f5e62f019883f2072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811391.43/warc/216197ea3691669858fb53ec0618ca74ebe5495d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811391.43/warc/efc72c8a6565c1f587f1f4cdc99ffc4744ff4126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811391.43/warc/fcfa277deb764c01ce914adea0541e39a46bce50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811391.43/warc/ff46fd537a69be02c7e7f16a14943941db33055e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811913.46/warc/324409f2530ce383fbb8cf7662e026ba6e64ca5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811913.46/warc/b591ff53aa8e3824a0740f9f5a95cf06b6a01c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811913.46/warc/e2bba5baa78cc2b378b9adf0bb894e0571dd1970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812662.69/warc/34f238944a33a55742ed8c2644a087441237f276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812662.69/warc/3ac6728560cdb270cdcf1b45e7998f0f055b94de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812662.69/warc/8373cc7266f7c9ae5fd7f5aa27933f2bd936507f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812662.69/warc/c66f55f8e6e04a6c3c291975e4157842bc71d444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812867.24/warc/010253e95eff9cda16d94fda0c174d2c248e3b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812867.24/warc/1f03d8d5c3c29f1c55e72d9324022ef95f4a8c94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812867.24/warc/4b455bb32e11e529bbb5a54e40e13153c09ddcd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812867.24/warc/e96cbd58be7bef1b71a7fb591d2b1f2a80f8df11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813241.26/warc/02c447ef7a105d0008aab3d50b7264f3c8ee8e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813241.26/warc/42b067a6d33c2e4670bd94eb593de772f658439e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813241.26/warc/84d84e7063bf3eca24f5ab2c4df613ace2d76b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813241.26/warc/e3e6614fc7e83f84ad5ab2629d52328aec1f920c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813887.15/warc/1b29c606d941ec7f59d4044d793681a5a441afec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813887.15/warc/3bd897ad04e75db4c4022658192667513199e24a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813887.15/warc/7f8aa66936fe3d6825bc9dcf7125ae6736a9b673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500814701.13/warc/6527a525a840d27e910a8bc6773ab721dc9515fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500814701.13/warc/75c4b0d827e8f6c3058c1fcc28586a187d132b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500814701.13/warc/7ac5aa597ebc6488e7690c471acacdbd2698d549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500814701.13/warc/fb90d4d69472e2298c9275b0fc75decffb2d3ed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815050.22/warc/1134cbd7eef11f35fc5d64a0f68debae027666b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815050.22/warc/472d6e2b91dbb13661a83c61c418c3ed905a654c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815050.22/warc/6b6406bea2f6dbc64ff27b2153920d2c425d426d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815050.22/warc/ab2106db3ccc01f1880bd9fb4585b23fd9698fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815756.79/warc/34ef3bcf1dab57473110926effced18ff30e80b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815756.79/warc/4c0886b351c1d3f634c8a902d290f1a893a93dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815756.79/warc/5764147b717eb1bfda854f7f38523060e721cead.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815756.79/warc/fcdffc9360adbc118c8c36e7927d6b1665fd18d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815861.64/warc/2244146d8276e542c173a778944083fb06f72f20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815861.64/warc/7caa610330e5b6bf2f75951704417c502a547d3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815861.64/warc/d4e3afd267bb988839401ab6d8e9daf5a5124d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815991.16/warc/160a2d9f8ca885350a5679c4ac4572ed17e5ee92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815991.16/warc/4b600ee1cfc6426559e3d8fd8d8a4c84090aacb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815991.16/warc/6a548f43f0673f2ef246b4acd36f035a74730195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815991.16/warc/6afbac4979682fde3bd084379fede35a705e75a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500816424.18/warc/35397aeb15c81e46fc68224a1841a2f649ab0303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500816424.18/warc/618aca9b1b6f36afdbab1dc2130fe78bb9a30a41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500816424.18/warc/be7fda04725cf210259648983adf2184289022bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500816424.18/warc/f88af9502c8adc4a5a7865fd77c32be780a6dfb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820617.70/warc/63f4655a6648f044b74bfacf5797ce6bf9faef07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820617.70/warc/df976bcc24237d89f9336a6f572579062ac30ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820617.70/warc/ea2be38307925e29b82c027ce7d3e939682a50fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820617.70/warc/ee0b38b73e3f88e36654a1c1456bbe908a9147ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820886.32/warc/055e362a61d47261ee77eef86c5099d86002cad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820886.32/warc/b392913e6719864cfb1a32edbb5e39aa4230c1cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820886.32/warc/e1dce94030774fa9e2780bec9b45520b56a74421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821289.49/warc/164efa540295a80fb5a2097d06d6a13d45dd010f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821289.49/warc/3c0633ecc5fadad92532bb534da4b47a244792c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821289.49/warc/c2e5cd09a8d0caa1897bb7d19d42ab2f8afefee0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821289.49/warc/fe48dc27c2b0588a7efb32a70288e0bee703d24a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821666.77/warc/0fe42a092589be773c5c4bfe54b1c007cdc610d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821666.77/warc/41ea4ad39ac329e944f0e66e303d2a06bb8a0ce0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821666.77/warc/460db4cd6d01ac5526017040aa99f45df66f50eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821666.77/warc/d2a8384741f17ff15d7eaf066a09a89ea769b3be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822053.47/warc/22addb9a3381b9a2cb4578cba93d1367f51d3387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822053.47/warc/d2546e164dec1f7a2351ed9bc95824216d2956a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822053.47/warc/d9c201d959b22daf84958c15ca6423e7d76b883b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822053.47/warc/dba918be8e9bcf6cd7bf9e25acc017cd2780af90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822407.51/warc/a8b13086d6513d34c23147720351aaf3ac8c2e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822407.51/warc/bc0c1e58933eaa7367d7ec40f8616809de19fa26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822407.51/warc/cf976edb80f548b43c3a020871303fadc2f7f9cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822560.65/warc/635da12f966184a6aae2b2ee757a5ea8441f6de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822560.65/warc/7f4149202211e9e9c1825e0853337cb0cf707e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822560.65/warc/a07353443e3553c4335a087a6d615f2625372b88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822560.65/warc/b9ef2550044334201629002a773c568dc608c3a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823169.67/warc/140facf157fa5d3b906f05fe3124ec4b76e21f8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823169.67/warc/3d879dc585836645138a662ca19233948dd0a7e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823169.67/warc/540dd82200cd6402f6a3a9c79b443ee35023350e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823169.67/warc/6fa84c7e0341fde495d6daa2fedb4f590ebfdc10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823333.10/warc/181d8b3cd5dfa8d8b9da2f4f6e8e68f96dd4ce6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823333.10/warc/52d76cabd0dd082a9a1e46ed9d5e63e9a77078d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823333.10/warc/a1bc77734c2ac81ef155a756c12052deb3bbf783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823528.84/warc/072f0e9954030d15795d559ace46afd633b3e75a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823528.84/warc/49686601b67f09c36bc7f038af649467307568eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823528.84/warc/7c6fe9d9656a710178ae8d6688cb060b922c8ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823528.84/warc/a174098a5a788c42e2f2b255691ef9056a6e0bb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823598.56/warc/0cbcfcd80b6e6534d4b0866ad6064d564dcfbaa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823598.56/warc/5873eedd2efdb7dee278ae77aa8c015fe95a5797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823598.56/warc/839cb209aedc4c9a3d56b1e339ef14cca5ce0e00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823598.56/warc/d9c4907157a211b8a94f1c1d7ecee95b499570e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823634.2/warc/51359162e8174a7c6652e0f9fd6d4fed1d689575.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823634.2/warc/56b663707c0d038d9506aeb375c7ccdb72492d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823634.2/warc/c108b009b4187df011cc321801acb8acab4379e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823634.2/warc/da36577120ca96d0f98295748415afb0403cf47c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824209.82/warc/0a66c5a228564968510a73489f297d6a36f51377.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824209.82/warc/12cbcfc9cf8f334b875bbd726be21774c4dd0a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824209.82/warc/adaef14163524f6b4c59f53fa3559e77916609eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824391.7/warc/655fd83f645daef83c61ad5e25179fc79456b63b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824391.7/warc/6b8a1cb002c953b0c4b315186322b9e93228d5fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824391.7/warc/b0735ae0624301d93a56079270ab0bba34ae9c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824391.7/warc/ef127e7669f86007d77913c734843baf815e2059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824445.9/warc/0a1f4e64ee6fe593d4c4876ca5440eb4cfa6aa0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824445.9/warc/26aba7183a0830733538931901d2f86c02659a7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824445.9/warc/2c9fa6739e39b70c1109298dcaff56ed0f8b41bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824445.9/warc/a2a2dbeb56a85877cc5c27f21735f518f99be3b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824562.83/warc/263e4e06faaf944b2e293f472eef9dd5e08f3744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824562.83/warc/5ae361891babd247f1a2ff6cbee79b81ff44cf05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824562.83/warc/9045a3ac307e2c68832339ae68defab83d8e61a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824562.83/warc/d21b483767c9d2eab6762eb2dc7255ac1617f6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824970.20/warc/2f89b42e8453fa8a72e4820c38616ae78d62899d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824970.20/warc/31897813031831dc6f43a55a1db8e4651e4bf036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824970.20/warc/eb00399adfa9c447d2a17f17e8f2b87cb7b1abb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824990.54/warc/182076a9bb1cf25027b35418f105e05c3414adfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824990.54/warc/9087ffce398caa3f1e754f61ccd63c22fa28762b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824990.54/warc/b7c509d2c5b240f7e4ee3df3773c649449378e2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824990.54/warc/fe8fab5338005281ca9087758171cb613dad6999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825010.41/warc/42530cf969612e0dfa28db39bdf151d82c142dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825010.41/warc/84f76abd7acfe92f806f1e0abb7d98d1a1ae085f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825010.41/warc/e49d3d67e028c35e863842f2d55bedd89d68d7ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825010.41/warc/e6f9d0036adfa9818d6117124138b30426164b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825174.42/warc/48750c44ef5739bf110f49b3c47c5ea960dab0d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825174.42/warc/78fd4bf54526aee75c6afa72a12e582ca77bd2b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825174.42/warc/d0fc46b233d1f5de35dfcaea83b133a5de28cee6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825174.42/warc/fe4fe31168f4618df7fbf15b373d9b37dffc81a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825341.30/warc/0477694b0d1d76281349b7dea290de8a2c97b2ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825341.30/warc/24393702e9bde24a74545fed64aca88ef919eb42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825341.30/warc/77558282f74cd12e36c897629334ec8d93593439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825567.38/warc/223198b7e293f7d9658d99d046fc523e1aaf2498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825567.38/warc/6ee954411b8d6fec0dbdd48456eeebbb662e69bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825567.38/warc/b276f7b43b802404c4d7b39fb9ea27a64376b98c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825567.38/warc/efe4effbcf36b205e79db0b4436e677019bec42a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826016.5/warc/0834abc7c4f2cf89cebc33c0dd8a1c693da145ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826016.5/warc/1c78ddc32ef80f97c383f3829e6a37c0abd75ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826016.5/warc/482615ba877c1ee6d4a9e5d78928fde5395f0d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826016.5/warc/7776b84543e559f7242ddc396ea871db99d65dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826025.8/warc/22c5aad09eb04a7487f917d588d56768455779cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826025.8/warc/bb4745ef094d3aebfe9b1c8b231f294dd521cb9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826025.8/warc/d139e75224f138f1a0f63df30bb96ead13e79abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826025.8/warc/df07ad367aca1838c291f66f44e1ccfc9309f015.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826259.53/warc/061f5f665b652b269797a3f1c3c9950d1fec2ec4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826259.53/warc/15f17237f1528c7354fa07f7f2f0b7d0cba2c890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826259.53/warc/cce1cfe3bdf1c2414efbf0be36588d2afea997a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826322.0/warc/1f909dba0c57acdff8ad5f488b0ede6cb7a0c952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826322.0/warc/3949ebec8a9f47f3c1287ab45d3f0e468e3bd6e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826322.0/warc/79e5ba0a68bfd459a4db679c63b7b0e5b7d55b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826322.0/warc/cc638088e554a80c67b000cc569cb522ec8ccda1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826343.66/warc/2a80d8be3995e4f06c1eda1b3dbed34fbaa0fcc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826343.66/warc/4b7645c7e721ddc44c3c8e57ef3a3e085590e621.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826343.66/warc/4defce3668518e4fcfd24667b746219754e9e54f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826343.66/warc/c5638e374449c49ab7896fa4e1f03d4f53e39e8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826679.55/warc/1c95984871e0c86cd64d3baf0fe7ae646f5e77aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826679.55/warc/2a686c7aab06f9b8a9c6762249e0395367d6f2f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826679.55/warc/e3c155fa9420fa01e52c15da69bfebe332025c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826679.55/warc/f272a3642a2ad6b072ba849be732d0b844717f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500828050.28/warc/2b07b5e07922c6f3d8b495e5d989e4271bc8d57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500828050.28/warc/e7a7ab2dd670a7a6e75e888028fa16213720c187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500828050.28/warc/feceabe94cd888fb47cc496af0625966ff68587b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829210.91/warc/14ffbbd704803f394d7b0eb0308fae8e7641a0c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829210.91/warc/44843c0fdc9bfe1fbd85911611cce7ce62396c25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829210.91/warc/643bc3934d7c7a28072a7f7c4258e81f33f8b15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829210.91/warc/8b6589d52a74424fda2827858c7a3123717e420f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829393.78/warc/197734cb19db721243e72f3e163b58b7dd25bdd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829393.78/warc/46c0fd90fa8aec1a4ba594619cf2ffa17e503852.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829393.78/warc/90ebc7e5a2193aa6a59ca6d21cf96ff8b56c7966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829393.78/warc/de4d71795686950f6c6583703db56e3a0d862b5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829421.59/warc/7e7a0a02a171272d156471759023452c10a5d205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829421.59/warc/cb314eab9a6acefc68b00d6ec990e40584981289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829421.59/warc/d4410d461833b761304fd3f2f9ebfd0cde19cc4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829421.59/warc/e168827c2c18dc8f00fe9b3d940839e466f2bf46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829661.96/warc/1f1b59f1811adf7738558f108db9a2ac879f8630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829661.96/warc/2cac7752432315230dec757733e293bdf25911fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829661.96/warc/982a60712b8f10a2917bfb26bffad2a85257fedf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829661.96/warc/bdcb4a657ef6cd3c771dfa34562b7af0d853af0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829754.11/warc/175983b0f648ed4754e27543525f361555c52a43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829754.11/warc/1fdb249a294b7459b02e1e84b26322f9fe48d850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829754.11/warc/27644b64a421b202e57dff7df39af5edc10a2460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829754.11/warc/37e32de02466a28ccfb3aa8a3389b8d0d1df9b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829839.93/warc/796b806232c43094b56d42b5e18b38c2a65d0b58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829839.93/warc/8d87214e1a564826f5068141d76ce0b4cdfdf2b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829839.93/warc/b6fe9ab9a57f14f4f288e986a91721e2d1a07f79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829839.93/warc/b8466c7a0617ff87a8856644897284d86b617c3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829916.85/warc/33defc04e201d6b70a7b0224404eb15305f8343d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829916.85/warc/591984bcca0332e300fb2f8ac4e19e3efcd50424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829916.85/warc/d27a5f49a180f83603bb62884cede9f5f9f0fa34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829955.75/warc/08e6757d21149f1dab625baa418ecdec8fc84800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829955.75/warc/1899da0cf7fa258b4a20b18a9e54475c5873c5c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829955.75/warc/5ae3c69b10797278bfd67efc85bc2d71ad81268d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829955.75/warc/cb06a2a2d8e6e1b0cd17375eee5166899315749b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830074.72/warc/854acd820344755cf4868992af1d591c3f996e29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830074.72/warc/9d76f9f56b65684b4b5c1d31e258dcbe167e8e7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830074.72/warc/a829989825bf7aea464ee75f5b4431be2ecef8f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830074.72/warc/ea44c67209bf70ad9cbf323a43181aa8ca4656da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830094.68/warc/17147cc3c4d833ec21112272099bd361feb45505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830094.68/warc/441269396bd41b6e1e3be845654094180625cb0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830094.68/warc/82718f857cc5436930740310628827e2f6bd6075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830094.68/warc/ab2500d435031308edbb6a311d9eabb85de11d10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830323.35/warc/9a8f61729ad683df0025f422e30629d151ce565f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830323.35/warc/a26ae131c01b78b240a7bca16cacfad2f374227b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830323.35/warc/bf992a32e3fc95625c8b3327f41c0405716465c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830721.40/warc/0e997393de3c3771786788341ecddc642753c134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830721.40/warc/4894b8e84777afe82ca10f7e22c2a72c04e9d83e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830721.40/warc/a9c266492c068c2dc6426d3a10abb686209e6815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830721.40/warc/af77c86688cc735acfc9dadf5d7a59f89783d7fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830746.39/warc/658b07a67a827de7c0e22e66f53cd582c25ffdf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830746.39/warc/b850d8351036c71a1380a1ce5d873c38535263d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830746.39/warc/e37756cb6fb9d58cab30c755452650f27609cbc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830746.39/warc/f490fb9d44389aedb2b95f37103b5118d33b286f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830834.3/warc/2fbaf0e28a17d2df700e71317b5b23b1022bf19a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830834.3/warc/589482ff7f95654948059b7da11bc65385a63bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830834.3/warc/6aca7a8e4be2df75373ae06e381920761d2722f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830834.3/warc/800e6b6c3932cdf626de041ea64abb9430764cef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830903.34/warc/0819017139ebb6465932cd84afc61eb74439a0d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830903.34/warc/3452a6ce97ba0e667b2274095b7ed8986d2ad14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830903.34/warc/47cc79f0f8ab62fc2bfbb3501c22334d3959ade2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831098.94/warc/2aeec726a874d53bc1c8210031d7155e22dc6ce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831098.94/warc/3212dd3901e080654fc4d534773fbbf20543470f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831098.94/warc/a9318916c34d26ef540fe9cc96c39db6e200e898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831098.94/warc/f06181a01bf0ff6c2ef97511b237e139384302f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831174.98/warc/2d77892299046d967cb6cafd7ab797247b868c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831174.98/warc/418918a219884a26f1459a385c22434928ce8de2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831174.98/warc/446de741ddbd3af1f2d6d22721de99fbd579c092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831174.98/warc/72e453a13ea9dc6604109d0002b2037d7398dcd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831565.57/warc/4da5eea2882a1ce67d1ffd86d798565519b392df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831565.57/warc/684013d94862670ea454be5c7723350f4ed903e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831565.57/warc/b89047260048eb4fd7d2b6b1be257772398cfd5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831565.57/warc/ff26ffda910281cd580a237b3861b1eed5879994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831903.50/warc/65b6f19d3227b17af0495880de471962c8c6eb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831903.50/warc/7a01b1a97b1f89e1f0bc30ae033a42b67c5b8e87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831903.50/warc/a9fdeb1c2e0c9fa708b62f2e020807ad4830743e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832032.48/warc/1bf58e6960d2ff67e0f0fe88566a05e46d95c685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832032.48/warc/496a5f8ede53edb883719982d1ce5c5e7f40cbf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832032.48/warc/647c8f30d2be5d6e62e401391bb37d6683b80a36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832032.48/warc/86f73809df803bb9d6a2145dd5b6824295ed840e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832052.6/warc/795e448effb5d49d6659bfbe2e56747d26ff8c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832052.6/warc/7f622cede53055e13aff836cedcbb713923cc6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832052.6/warc/a4e38c1bb60cd82a85180a1ff05954e973d92357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832052.6/warc/c005c65e25fbd75249a6ddf0d9fd7b93e80a3557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832155.37/warc/3ebe8bc88690b56ad9ed657fbceb3b752e29d837.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832155.37/warc/aa22e0bd6d69800d6ed81479efdb8913277859f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832155.37/warc/c60dfba2c42709f1fe6adc28166b277bb8594d21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832155.37/warc/dce79e8df6fd5dbff3d4fb77b40c4ed586d2a8f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832538.99/warc/052488b50f4dc71039aaf2b72f062b447717cec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832538.99/warc/6cc62da3e83b39d1f204966951ff437412a2a1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832538.99/warc/d723eadf5c9d530f40bb9b1d6368ffc4c00acd18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832662.33/warc/236d04d9a14a329bbb1b32d39d7e79e10edb1e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832662.33/warc/362fd55a13d88ac1042ffc2a6a0c564b4576ecf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832662.33/warc/72b85e7dc3926cc267660c639578cf75e60d0209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832662.33/warc/bb3ec8ced20e8e4c62cec375971343add85ec760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832738.80/warc/176a66d62a169f918b72170587dc1736ae627268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832738.80/warc/37b7e6e2cdf6584531e189e373b23b884f57cb0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832738.80/warc/9c3f62006579fce88e2e443320d156ac915abd07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832738.80/warc/e6d5340d27ecdb61cf7980ccd2195829fb42f2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833115.44/warc/4fdf6191af7955dcb929201e165adaaa565764a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833115.44/warc/5fa81117ebee0f0583832478692793b4c2bcc74f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833115.44/warc/9aa084618d209b649adae0ad3a479103b9efad73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833115.44/warc/ffc418d3c5bd151977f0abd3f6825ac89453532d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833461.95/warc/2b0e4ebe48a072ca93479544a907f8ec93ca2239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833461.95/warc/a8e95a896d9d66e24d77c6eab0ec5827ded5680f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833461.95/warc/b5b18c80db32f94444c8163902e31a70074e7afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833525.81/warc/484d377c544a02b4eb859b76bcbce929bc4538eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833525.81/warc/69d7759719aee58e7198ea590037cf7f7a373c94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833525.81/warc/f97a2770ed4bfdf6b226f3904c3c99bacebcea61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833525.81/warc/fe46cd41cbec0aa779849f280d33ae7101dcedb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833715.76/warc/2740081ae4efd8f32fbbf3ba7640fcd3bde177ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833715.76/warc/afbcae87e62534bc9af9fce60cb718054bbb7885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833715.76/warc/d47d06e14140796b602b323a3ef17411f8a4a725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833715.76/warc/df144406f3ae96a181a42c36cf14fdf9f5a9845f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834258.45/warc/5ce213ef7b506c1259faea0b0aae47576095cd9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834258.45/warc/83b63c647637923de50283903ed2fe41b7600f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834258.45/warc/8db2a4b0259b3236130be2ed06141b127f5aa4d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834494.74/warc/0623e155ebbbd5a593b8d6a7ab34aad825b8af10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834494.74/warc/6b56bf7d9c44e1698a99b52a11f726ca850c5133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834494.74/warc/9554fa9e5db54446891f8d69fd92ebfd8d5e5e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834494.74/warc/fa598b47da4ddddc35017684f56054e40b2face2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834663.62/warc/1e3950110c57e4667425a9978fc6dc4cce789b66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834663.62/warc/3ee33068241b0359bad7ae1d97962a397ec483c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834663.62/warc/b40e9cb553e4c7c211ff2bbdb1afa7b6dd2e503d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834663.62/warc/c1395fd05c1fe34319c4025307005cc5499cc6a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834883.60/warc/0781b09fb375ec71dff08c0723f2344ecf8ad078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834883.60/warc/0eb30d88e661e249e73fa7f8a9c90071cce7f23b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834883.60/warc/d170c3c1fb79cd1ddc596706c5290b8747066f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834883.60/warc/d6f1a8a7753fd21c70fdd80ca493ea1c02d97da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835119.25/warc/1700c75fe7e57e23a3150e71ae8c12964b3b62ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835119.25/warc/24ed4546153921b124913b42fdf6dd23d7a6b28b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835119.25/warc/b9744754b573f3486c631b5206e9ad27342feed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835488.52/warc/27fa59e45a5d13a88e822648f59c801624b0d5b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835488.52/warc/6275045abe65d41d13d7fa1b584772059a3d7935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835488.52/warc/d3e5ac3f4e7fa173da55ad8a12a0c06245fee287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835488.52/warc/f59343ffbf4f5e12589f95de9981b1247d1d7c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835505.87/warc/60004e582a874fe1cfda49d8b9ba901a69dc9c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835505.87/warc/601e024943ab54daabde39dd36b68dbcf33a7141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835505.87/warc/696563454d079ce898358cff00165b6aa468ab7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835505.87/warc/75db68ec80131e92fce83b682f86e5d2375a3019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835670.21/warc/40b3f7f6e89eda3e1bae811046bc38ba4e7a89a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835670.21/warc/5d55909aff9777e20c43595b1ceb49b5b1ed4489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835670.21/warc/8111aacb5a9818f5ff0e8dff09ed685c0803932c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835670.21/warc/cd16bb55efdb1026d14d01787ce37cbace0300f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835699.86/warc/5353dab9dbf6ba6b038cd4b6adb130816205ab05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835699.86/warc/6c684bb618dd5e957a08ae33dbe2e04ffed7e787.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835699.86/warc/bc5b80aa529fcec8324313ca7fee84c07a8facd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835822.36/warc/9102a8bb5802c644bd97ac564a89f9a1ce0a9392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835822.36/warc/92992daa2e4809227e7c29344f416460dd25aa8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835822.36/warc/c1c39307923af971b728b280c130b0768d0f6da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835822.36/warc/f2eab9dbe27fab3e69e31893de15435f5ebe8aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835844.19/warc/0dd968bf8bf0c25416ceee1068473d8adb47f8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835844.19/warc/6a393e9bf4878c68716e1f8ab82555bdce90478d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835844.19/warc/73da7a708a81fffea2e5466ea395885465fd0908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835844.19/warc/74f5f6e56c5497e2c6c96a6f29eb40fe436d57b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835872.63/warc/8d1846596b87e948fb07d59e4d3a8163918467a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835872.63/warc/c1d9e5296560190fde023267490bff62cc2d2b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835872.63/warc/ccc16b00961de7d8191ea1aa45f3a0f19b0cc549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835872.63/warc/dc3de59a0ab829fe6b05659a79efc00095988a7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836106.97/warc/16bfe1accc91f8c3f7b90f103d787350ab2402a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836106.97/warc/a9d40567d36458efc5dc8838577fb5bfc0212553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836106.97/warc/cef6a1ff96d2815b7ea6d80bf96022f3b917836e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836108.12/warc/0409df0899b2a7313195f2df2a3a6555ac47ee9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836108.12/warc/15ba98b11fa7cdfff4808eebfba7bcf184e2123f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836108.12/warc/8eee9373003b22c7457db6e6ff53271008633f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836108.12/warc/98e0bf85ddde3c1bf944c0c786e81254ca2b8f71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500837094.14/warc/43d997c3cc1b8e0b567da1003f2574ad5331ae00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500837094.14/warc/4cdc85f429e730a45d4cef2b560c3c9928de2de6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500837094.14/warc/a77d32257818061affac437a52b4b4b5253ffb9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500837094.14/warc/f41da9ed41f8b4e5480abd6f569e1b49f24b2ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500959239.73/warc/03436f67172b71971be8f9178be72dec22a34848.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500959239.73/warc/5d3b59b05788b6aae0726dfb5ea7fd35d00c2984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500959239.73/warc/72598c92a62ec44dd557cb0358b739c07d143ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500959239.73/warc/74c68885a9d62603b0a65bcc51b1cd7b3a383d57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917463.1/warc/35377153167332a55c20fe5efb8eca11967e34e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917463.1/warc/c18909e2237c13c06ff4703cc11009e87edc6c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917463.1/warc/c40d9ac98d6e3eb48916a353c94bb542a9b1f246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917663.12/warc/4d3cc3d0ab1f89a646999e013070280e50fdd364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917663.12/warc/bec1838920764a7c373a4e991833db204a10a07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917663.12/warc/dc9f9a0b3af6e2b10a5de296317193fa51330e2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917663.12/warc/ed038c5b076bc2f4c47b311de593649be00ae69a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919066.8/warc/06ea2067a9128ac2b0320f485c8d98f2a7c542e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919066.8/warc/3ad5edc740ae950fc6d031e7c33da65b021fea02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919066.8/warc/75a47a5426d63b018a7f8800d8efc8c08a6fb8e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919066.8/warc/fe31880d3705f1263db6c18b57672fc1c6f9e380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919886.18/warc/2bae228ebf3982727eaca5f834961f651a587d3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919886.18/warc/4e81843c863a14521ee543ca6ddb8a6158429bda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919886.18/warc/ad582ff52b05a62270cb614a1f44324906bdb6ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919886.18/warc/c05b85ac09e9f13a2a3b6c16978a7dd2da2536ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920694.0/warc/3d3d4f42f60102051e3a4f7d42479fc3df56ba46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920694.0/warc/92cb0f0bbbc1d0bebc0177b6e4c0b9bca6c5d077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920694.0/warc/ac49e621a1b73dd9437cbc3aa04407ff320dd76f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920694.0/warc/e38dafcb3c4730dc74ec4191d3d2b2aeecf3e386.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920849.16/warc/a9fe4ef310a96048691622e4aafaf7e282257106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920849.16/warc/c1661de9f07c5c447b0ba276ba775f575c1a1b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920849.16/warc/c5d1eda3861e35acdd25c934fd86f3ce44de0415.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920849.16/warc/d75f84eeebfb57b96b7326b44aff4bc12d96b657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921318.10/warc/0043f9c6bee66d6d1549e1827af189c36d4d3653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921318.10/warc/08a8159645ea0c93a71ff381c9f49f334a78c571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921318.10/warc/eecd0059da919f01a50785da53b6ff25ff1db531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921318.10/warc/fab05b0d2b047a696a9a1be2e763f97ce2575ec6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921550.2/warc/40054c83f1611825a985af400d330d9d8e69eae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921550.2/warc/8e1d0acfe618a2bde0da572cd429fade5d24f016.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921550.2/warc/9f94e205cdedae827451e3dd4d9a6a813bb036e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921550.2/warc/dcae9407c7498706ef28732c6b2cf04b596255c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921869.7/warc/0ee98568d408dc26e969ea5baa41f701dd9349df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921869.7/warc/27841aab32575ab5a6718f0563fa5ab1dd4b44a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921869.7/warc/75ebbc178d9f54b08f94a51841fbd6c9056d02ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921872.11/warc/1e74d271fe884638544db8b575f80b3f760d4e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921872.11/warc/254fb2faf11044c9f85fffeeea45e9f746ecbe39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921872.11/warc/95d5ef5c44d47f1064f37646fb173f839185ebb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921872.11/warc/a18259086ccbdecfb3b2d160d2156a65749c4d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921957.9/warc/49003a302a602172a224dfd3a246c743e899c6eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921957.9/warc/8ef924bfd0b16ae491c1149ed778bb0858d1bf67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921957.9/warc/cfa74340f564c7fd4024df1472c9d48341f3a275.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921957.9/warc/dcb196c2fb7ebf597eb4e1fddcfb43771ce928bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922087.15/warc/01b73d58f155b7cf642cd1526115b6003ac13d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922087.15/warc/14f8f91fbf68221162a76214416f90a15fce9f6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922087.15/warc/3e80d593d253a72f88084ba15dd131a95bd92881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922087.15/warc/56540c71bb3bd3f289b5d0ba3dbf9bb0c1fc6087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922089.6/warc/404de41186bcb25c7a45f7ac1e4dd9ef9aeaf434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922089.6/warc/4e5579b36bd9ac27fe5bf096b96484a34e40179f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922089.6/warc/a5b0ee6e734124f23d86c8537b4e0cd1c5a6bb56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922089.6/warc/b16d735a10c1a6d09fdd647311ba3c44d6052e9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922763.3/warc/45a5620522d97a05244c6205ed19a0a0b21f5c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922763.3/warc/b08cff7fdc4a4b8dc20833fae3e89d5171402d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922763.3/warc/cf370c6ca1de6ce137a755fbfbcd60d05bca6c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922763.3/warc/e30aaaa5d9496da19aca050aedb8dd7bba55a0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922871.14/warc/227533f7fb14456a840cd0964645fb1b186787f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922871.14/warc/45a945c41c90a656de919833f18678e999743a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922871.14/warc/8d2ce2069b9bed0b85f77e939bd2f31e52ffbd0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922871.14/warc/c561512563683ffe512cc81a99b232344e6b140a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535923940.4/warc/2f2718b3d913f0bc591b3193ccb982fc4838528d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535923940.4/warc/46f4d60c4222edfab8b13c4fd2919aceb3a674c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535923940.4/warc/5f5a7c69a626ec86e85caa14b47fa778cb7b9c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535923940.4/warc/6a0b25faa066f2330766a5d76d9a4645b96bc7df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535923940.4/warc/a3f9a3fa2f0f06773186800eaa68d84d58661b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924131.19/warc/3f2faf29815bf6307a1eb6b23b887113f25f0b82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924131.19/warc/7d7dc8e669b8d3fa14d976b3e56e3ae23ca98008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924131.19/warc/88bb030c0f9180fc5aa4ec62eecb7fbf700747da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924131.19/warc/8bacf04f2094500a1bfa1f5135fe8db9ecd87ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924501.17/warc/4206689bd78e1ab17cfdf2fec08f184c5fbe6b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924501.17/warc/c044b12adca9aca13c755e099b783993d086a72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924501.17/warc/d856d2ad2817f609ebb8d2c52712d61cbf78975e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924501.17/warc/f8ae70d68917e9326e0cf7c9a61a24290549da95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535925433.20/warc/422f297fa1c84b26eaecb6b2bbf19352b265fe29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535925433.20/warc/9aaa3c1dba90abfef7968b1d739fbd6f007584e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535925433.20/warc/ba92a9003692cc0732a27ac2cb1d8baeb528ad49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535925433.20/warc/bba045aabdd11278de991ff4a40c6bfe624a08ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535925433.20/warc/d3412e37459f6cc9ecc731167c95378b9ae24afe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/17f2413f43bbde0389803a7a7206c4727ceb2988.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1b99519eee322bab862293ec750daaacf504a5e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1ea9687f72d22bb43d79cd0a79230bddd3bf5afa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1f91f81e4cb8c0f4b32178b147e02e791b3bfd73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1fc34287075cd830c9ee1be37ec1297232b74588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/293d995e5ee616cd5a86aa61cef13d7315b00b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/2a09d372d7ae50670244884952e3bc4f06f81a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/2a46bb48e992967e26c8bd46cf0e62888a395c76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/2c1b706d73d89f96f64a2e004c811a8cfc44bb39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/313470c0c45f510979912f55720e693e5c5d49be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/31aea39d8906291badb0ca2f9240900a1a7c3ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/32e2cf815fb5a1b22e725accd085b584aa375d1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/3740ef042f2f9ed104bbf231dd568308039fdbd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/3a56295a600f45e8426189452440ddd8c263b71d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/3ecd2f5c78ea26da0bd0c68b644ded8422dfa92f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/41cadea478dfed9018bdfd307334352b8203525e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/41da3ef19cd26be8689ad1145493c0cdde72b1e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/42cd1159b58dfbaa3bb8adf0d816092f14fbe008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/4c43b9a0cac78a41916e1bdedde343fcd148225d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/4d230dda3c4e8506115d76be3726444a4607a35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/4e1f1a66ac0a28907254433a47d3399899302f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/50230fe073a153b2e6f16074eab9f80bc2ae7a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/535f8c04927759cbc24e5f63c991a8631f509676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/53c2bb1b2f12de17a67c2b185d2e8ad0ccf55b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/540cb922aa24f7dc8044dbab6fa2edd1b491db51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/5b5d291bebb678fd2f4206355ca8abe1689dbc9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/63eca566219b104c09c679c3856207d44994f226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/68db0011e1edfb6e9e01c536b515510bf38efbfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/6a08cdb0cce4180d527e10b8560d02ed8f6de2cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/6a8ebf1d2244dadac77f0ae72c9a9294c6e033c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/6ad72ae65cb3961088a222763033d12f4de8562b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/6c09240d782c39321a959e8d4d6748efee29d93b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/6c3bc5faadad887782403fdd8e07844b4a08da64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/7048197bfa282fdd0c4fcd6cc9c0bfd914f2a531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/71eb412c9db4e0019522b26230722492be618804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/75cf562584c07df50ee9b127fce9621c8e499e86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/79175165028e7637c56273a051a479fc528af402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/8414e4b633cc52a3a403f46e19f80b91a8a07911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/851498e299ba7d4c0cf5c9b08397b3a73221b523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/851b15c719577d0b62bdc2788210e4f6ed4f1d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/86337872b7e94356157fdf2d3caa7c8adf149729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/87ffc4c13ed3cab6f4016e3ed59be511a49a44f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/8aca9cd6e2b62da4a4b17b59386501a674fb0044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/8d6c1ab7d5835266ba21ecdb0c62943c9e018d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/8db26b6631c683fd100267cef5024370fd7dfa0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/8fa4514775ed53e91b51f17d89d5b86fbe772b9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/9390087c31de056bd3d6c976de9a0cd74d157b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/97edddf28f7c8a23065f5ca8f55ba1225850f254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/9be8c325c7258e875e2fcad955ffe5c759de442c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/9c8e55c3c9b88b34c6d115a4e1572078fe97cec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/9fbd9ed596a9bd5ea878eed6629749f330fab3a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a092d62f2a0fbf1a80072e016952f4cc918dbdce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a0bd08961a935755f89b823cbdf6e6fae23519f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a2993df1668506f50ea80dc9846eac4b3cc8281e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a2dc68a5cfa0331226bdd10a576d08ea297be681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a572de1e26e03d90eaa22d0e93df124231a097ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a8413a4f58c1e9452389cf4179ff41d13d41dd0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a9f074fada244fb31dcdd206e4e8f80da3356429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b04cc11c6238b65281f9267a426abc802b9e33df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b1af479d284c35bfd474e5f6baec31cbc30341ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b2249e2496c90062329211cf647dc18f303bde40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b55c49991f59e064d192aecb1ab33c6ba8cffcd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b6419d6cbf0150bded0f14c9314de6755cfe6b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b8b67e5f9e411228b5bc2d05b3f7ba0ca0e9af24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b9ced1c2f706350e90d7eca3694cc18400c6ac68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b9de763d3c37d207814c99fb76fdd9848a11a734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ba2db8811c84d43b29c3ebf558623a9eb02e226b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ba6a7e5c53e30b971085fbd2fa602b0191e57e5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ba6c318cfacede4d07d7ecd3f4d50cc4a3fe5f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ba8853cef476f4129ed62095d2bf80a064926cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/bd0446e1242e79b8b54a5a2304cb952e172e031a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/bdf6cfb04fa90ca235a33cfac3073270644c7f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/be82b1a4a21b8505121571c35de7b2b83debbd3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/c72637927b2cce84004d2d48977e566d805e84c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/c8b0d930ecb9d68cc616a90c2f0daff72beea01f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/cb0f3add9ec3e43bd2bb763ecf1c02ae6e74a882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/cb88ed7e4ec9487f4bfd5c28ed142774422d65ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/d2356bec5bc638d0d95d6a330b6c2d61ade3c110.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/d2834affcbd5c3e74a2bb72a819108bf0032eda0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/d590bc076eb8ce6166c27ad85f7476c60467c338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/d5ba8ace7ea54e38073c04b5166277f990a7e05c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/de4a3249ec37007a9a29a30b96f92715c8264216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/df48bf6a2b8ecbce0d695d0fd7492dfc754497f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/dfca7b7a5f78fe03713fcd79129cc9de903bdfd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/e18d7e9bd08b578e81996bae9c68c3ea23aea701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/e5805fd2c46f73b4da31ee89d18d99bfd6702e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/e7182939322c283f38c38a1e22b8e0ac9eca8823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ea13ad2cb04d38836eb3bbb5ba59be10a924e0ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/eac47e301480ed1fdc387fe976234c9301a9f559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ed02ea81749625f20556cee23967f46b18daa96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ef122c041e03d18555044b109dc64db973f4adcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/f04e55ab2f626da97cd7f32802de90bd7325b3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/fcc1b7dc2fdb5151531b20d22b09ea31ce7b6ef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/fcfe527294ff2ff59f7391c601dc93fc07d00ae6.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2014-35/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2014-35/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2014-35/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2014-41.yaml b/configs/cccc/dedupe-url/CC-MAIN-2014-41.yaml new file mode 100644 index 00000000..76032703 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2014-41.yaml @@ -0,0 +1,504 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/0228c75513f37751ad3f0450c0479b642e993c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/0249a76ddb7bf44ff7b5496859ba93a545ae9389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/05a534973aec4e0691b4eea8b09732083b450b53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/08560c52d1b622a58a5a8e143e25129505811da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/097fc55dca09bdb47dbf18ae0070c1eec5e692e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/0baa92682d59a7a1f9bea258084ec50082511478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/0d22199ce5b52b04922599539254f3ffd3c692c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/0d420f7320da6b5df827912f4ef6fc85b25ee4f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1214cd1b7f6fbb7e48d7cc34c076e7c754852926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/13e0d878926899ae729515f7490602bbf11f61f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657102753.15/warc/670cb56945705898830ab5b3ecdfb4502f306fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657102753.15/warc/759541d6919c323183999c5d7a49f0bbbd546d5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657102753.15/warc/d61f7f9fbe76dfe8a6142403baddf8662ee85baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657104119.19/warc/23fee1f14b08483ff26532862494e77853ce346d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657104119.19/warc/c9f07874891ae182af38ae18333e82f28d79dd36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657104131.95/warc/17d182d8cf2c148b8965a33f4f24ed2341329c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657104131.95/warc/b7b55c352cf3288b111a9521d3a52d21f4451fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657104131.95/warc/f485f715cf69248f58ab98763923fb6d3344bd2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657110730.89/warc/45e42e34782ea4086b652142c95c679da1a802a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657110730.89/warc/8550c90cb04cb5f6c1ca0d787109808cbc4e050a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657111354.41/warc/019bb6c8f31f0212b08a56dee28126b2a60cc11c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657111354.41/warc/01e6c3d1e3184b1359d9a650a56efb3f7d187e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657111354.41/warc/8a5026857fe7bcff5b9e2e591bc4be14108353f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657113000.87/warc/23c9b22cc43fc91a5565608defca00c40da1d6e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657113000.87/warc/bdba7fede8731e759d947a940ee3ad46d9e019ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114105.77/warc/0a406cb768fd19879de48d9ed765666dd1a909eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114105.77/warc/6534e82085f505e0ac9bf74926050e5819bade3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114105.77/warc/a0d4f24815cafb910284473cc65fc440c5fdebff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114204.83/warc/bf0eb3db11400544e874895a752f4e62025963c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114204.83/warc/dadfb58ff549cc90d1f32d619cedb93be6a2f127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114926.36/warc/2b7634bf1b91c4d8cf90acf138aabef4accecd22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114926.36/warc/5f391eb27b7e362f50c2bfa515f7a02455d76133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114926.36/warc/bebc57b16f8f0e4ea5c39cdb2ca65c6d112082e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657116650.48/warc/75d861297a18687abc711f126c243c323372d78e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657116650.48/warc/ec5f40b32249fab55761664a2d648c193dae7692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657118605.64/warc/0427cc23f1ba79c3a61e925d8a7b364ab010d6bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657118605.64/warc/075ad171b6916dd2f9a5de62412cf0eefe5a6e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657118605.64/warc/e894e1981c6e98a66d55a0dff96ae3bf06b48e64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657118950.27/warc/8424210af90df47caef6f270f18f6aac4b7e0d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657118950.27/warc/cc2e4d830e3d9c4d0b5cc2499a0565bd0809659f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657119220.53/warc/66b7601779d17adeaa09413cd3b50cb8525e8d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657119220.53/warc/739bd6150b85b00cdc0d29ed28695cd158b9a1f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657119220.53/warc/d1fe67462b89416ea8ed36ca9f41927e6bd9e5bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657119965.46/warc/277f00933226939d4b5c8c2cf1ab4134b8d170ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657119965.46/warc/60b00dba0874f3a57c1589eefbbd0e4cc37e906c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120057.96/warc/2ec4e46af5c795d1922e4852ce408734e9199c98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120057.96/warc/5d64370f190bd69016eb2a956e347492ec7ee3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120057.96/warc/cb14abc02ef964276d447c6224c7196567eb0fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120446.62/warc/82647b268dc2d73f5d5c6cd645093a4dd1a7e670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120446.62/warc/c3a9fdff328d983c11c48443a2e8fd1fdfef06fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120974.20/warc/659365359988c3a449b75b59d07fc7242e3056cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120974.20/warc/f464dfa4ad3ea4b217a9c94fdb7153275798e5c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120974.20/warc/fc74080eb0ce7b950224550f4d364fb142dc66f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657121288.75/warc/34aff26123d2142ef816a5b879da896f198a0f97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657121288.75/warc/6f27e565346a68a281b1fc4c62f3b4c40a16b7f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657121798.11/warc/f2fbe73293f27f911030942798a35a59dae02085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657121798.11/warc/f3e7c7f70ffa0bce7bde671f9819545ee9bd0983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657121798.11/warc/f7a7b74efcba916a97af92e2d142940d48771501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123274.33/warc/0f1259cb779eb107283ffc2e348c5ce4cf6f6e0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123274.33/warc/edc3962fecc45f6b2f252953729e2655c6115d6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123284.86/warc/5c4833318ed3d24054ffb57d778a2799ce842f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123284.86/warc/f314090f3cbdba5285d9914ed33dd766539a0831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123284.86/warc/f616e5b4cffd33e312c2349b199ed5c54be54a8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123617.30/warc/4c5cf78e3768c36d07b68be6892c520d89da6251.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123617.30/warc/8ee7c7a8ab8271a2982d16a861cead9cb2feb174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123996.28/warc/6910616d664fa33d97dfcdb8608ccc0905ef9078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123996.28/warc/d2009c1fb09813e237fa5418c68683870c65da1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123996.28/warc/e2def4afee58cb0b8817db2cd535f5cd892b33fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124236.90/warc/31ba841efa6f4e03aaa11701bf444f2b424625c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124236.90/warc/5f6916648653aa8fa56964d24d3406629015a82f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124356.76/warc/12f41aae5bd70912c9fa64c333849ce2fa30ab73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124356.76/warc/9f51162fecee54d0db38d61d524e77a55774a237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124356.76/warc/af615be9b77b93480d9c86e04778bd595625c4fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124607.22/warc/218c6f875f454bd9d89c4649b5af97fa7b47d047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124607.22/warc/5b4eae76ec058a0251fb9ec8db1a93a5d3c4ba3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124771.92/warc/b7eccd74fc4de428c9e91431cce4a4c7b0179dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124771.92/warc/c907743cbf4c7bec04a08b5159c11eab47d43595.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124771.92/warc/fcabf575c5b4b1f70d00b989b1d475b11d0749ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125113.78/warc/ed893b25afea0cc4c36a5455a86fac131fdc1f78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125113.78/warc/f9a0b15a4c514e69306680f96d190a7f53180b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125488.38/warc/373de784bbaf098315946a907adcd2380c8d41f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125488.38/warc/bb7f152588d85e455c23756adf7f5e98001c7eec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125488.38/warc/f0eb024d1fa0f79382b8627a6fce203b5c719c79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125654.84/warc/7b539086087947fa3804a4bb3bcc0a72c5743329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125654.84/warc/f5becd443d037ef4839aa3559aa1ceea8c581b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657126053.45/warc/600a39a08ac88fb19e47f555464bd7e0b2a13ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657126053.45/warc/cb2a5437a2aeacb16eb1ee6ac59da00de1f23f67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657126053.45/warc/f8b7bdb69166275d6aa6b933affe1fcb2a0b8633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127222.34/warc/0768518e2fdda662d0ed4e9e254010a4d940d4f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127222.34/warc/fdbb38e0d3aec4b0cae4f363a09effbe642bbcbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127285.44/warc/11381dad6de3834e90aa556b396a265732ad0297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127285.44/warc/7e8a7bc21482413a36d8a1eaef0e3e3551a0105b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127285.44/warc/d783efda50e9f4c7099b883c22bfcd319e75bcaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127503.54/warc/36046d9719b7ed1ea175c4fc45e2cd7ecbd4c212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127503.54/warc/a0d111aac1c04de96b07a20b1a5e9b71b157cd57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657128304.55/warc/1c5b00699db5c3c408176819432d6dd78a3474cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657128304.55/warc/354eee8c748e19347fe1ad8a042811573d881db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657128304.55/warc/c5e54a7d1c2c36c43d14c1d07c6597e8a1fbcb61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657128337.85/warc/4331492cd59ffa3724bc7f6ad6e2b38ba74e25fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657128337.85/warc/5bace44f5ee288ffe0eb8bf5cdb40c24480d535d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129229.10/warc/7f5a4b709c8d72996d0f0280b9e7aca91b8e2b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129229.10/warc/8a379a3a05a69881478034ab735266012fdc89c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129229.10/warc/c163e8028486c03e1afffbab6117e8eed802e88c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129407.88/warc/3ab4ca13b1f378b916a6c0e006dcbb55bda0346e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129407.88/warc/af53e416b259da6e66adc62061b96a878943f954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129409.8/warc/446766bd09f09eff0e00538dd03f240cf7ea873d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129409.8/warc/70479fdfc00932779e958a64374f46979f958f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129409.8/warc/a5e66bb983c64df42d5a8f577cee4316fd518f49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129431.12/warc/095882b7f43847f340cee8b58645388dbb3f6e8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129431.12/warc/4abe18b4f06b3ebf641c102fd0f5185a2aa84ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657130067.43/warc/1ddcb12e070307b093c4896cb10bd5aa7c392a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657130067.43/warc/33ecfa6d1d524bbb50e8c88028e4fec3323ede28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657130067.43/warc/e54bc6a295111cfe9cce0b062e3c68baf0298469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657130272.73/warc/793b469b5c6b6f3936f3433314491216ffa56ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657130272.73/warc/af029c90c7c618a30330537d6bd092a5fbf2bc9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131145.0/warc/0975abb49dc06db1206cc9b5f338147cb1ad6ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131145.0/warc/9753be172e6681611b829f12da0fa55540b0e02b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131145.0/warc/e1393e0704333a5b32857f1f6df067b9f3cfadcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131238.51/warc/15e38e80eda3f9a14dd07cb7e7816810cd12f3a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131238.51/warc/e8df21c3d3831cd8def97f3c6435f79ed2dec9ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131304.74/warc/6d1650e464c50757c77b1376295242caf829fd72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131304.74/warc/dd76ccd353b83f432fbaa27181ff5ace59c9fa1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131304.74/warc/e786b70f060a52822c73043b49f2ada1cea96351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131376.7/warc/06365deeefd0ecb028380890a16e9fb77cb70052.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131376.7/warc/2e601515dc982eca4fd7082a1450594d8e9c67a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131545.81/warc/6b9c78e2f45ab919936af990d6a219207e8ee283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131545.81/warc/9c2386088390c903101b8ae56e0be4449e3280b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131545.81/warc/f3d8481eb0457646dd7ffbd81dec1ccf122a9b13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131846.69/warc/4d8d500125cded777ec2d0b22e6ccdbbf16bb471.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131846.69/warc/6ac441f92ec381cc31299f27bc8bf7a6bd4ad84a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132007.18/warc/368567f72dae83cd57ae00bc8e17b9fbaac1c2d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132007.18/warc/5ff11441137a368f10445b8cef83a0b2a5784744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132007.18/warc/d9b1c1c4474fdda95ca6c55df2dd390dfc017069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132025.91/warc/5a7c0053f218752a8e702a747a179696e33a1561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132025.91/warc/d263d738e4093ca2faa364e6ae5de558d773c064.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132372.31/warc/3d0f938af23924fc5cb64e7ab7876374c46dce2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132372.31/warc/dffc103830f37aa9377cd5367cc351779dadce48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132495.49/warc/55a4187d4f3e02d1c4233d0fc8268138cac90947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132495.49/warc/57e57c29214724388bd3eabcc74d60a797a351c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132495.49/warc/763256511a755f7597dfea8b499db6366785c13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132646.40/warc/83e29f0825df8fa36609d1c97b93530eff2b6a04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132646.40/warc/b60669d4e2a49212237619fffd72fd88ad2bc027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132883.65/warc/1a629e28b8d269de9d8b90573a6a64df1bc1205e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132883.65/warc/b7c132f27b2fbf820f8c7d89f3882bd7b9230da7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132883.65/warc/cbdb1ca916bb4b6de2247c75d4f12d0990eeaf1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133033.29/warc/2f7e91dfdf77c590be42d2af92f06d8ac500a393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133033.29/warc/f9fe3e4651876a8e91016be63b5c739ca96f06ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133078.21/warc/2d29b152fdb342d40589e8f598d9e6941bb8f0fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133078.21/warc/6f30ceeb4c574e019aa680db017728649a565451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133078.21/warc/914a342f6c1b3a9c0e22001d65f32af38cde9444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133132.72/warc/d534c387836e7a46f63af86fdf29a0c0b53d4e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133132.72/warc/d9b6e796ea825b47a790dedea52b558f8b9fdfd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133417.25/warc/029c1d50a11436bfad51fa3bb09d97f1d1bea104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133417.25/warc/2c17b315b8ddcfcf47e293ac4be2576b23c6f883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133417.25/warc/a2d1db0bef4d6f22e0b912129ad6a74fb81bdf74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133455.98/warc/171f2c2516c6d3f2409fa24d5ec026643a437d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133455.98/warc/d44a009c5e19ff5eb7dee4c94acab22660a2bad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133485.50/warc/9428fb9ffa058383bfb72965ddbfbd6b6fdf888a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133485.50/warc/bb46403da8560a4f164530aab7c08e2ef1f489bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133485.50/warc/bc6330f1220a6e70ded81c3214c4f671547915ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133564.63/warc/c40e9f32b81893253c07f158985faaa775579fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133564.63/warc/dc3d87c7bfaa9eedb7609f17b604223097dcfc4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133568.71/warc/3025653a235220a0a76f8d2a5df04415bf67b96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133568.71/warc/428db30f7084f7a485d03e5046be770b948b3125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133568.71/warc/9b36affcb9f94b3e6517f8839de01a595a7ce7cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134114.26/warc/77943a490945d561c30c8f1b55aa19f08e855069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134114.26/warc/eed8329ac33c938f8ab6e31595f9f01ebf900c09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134511.2/warc/158f73637a2004e0814db72b6e3c4ce1d593e798.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134511.2/warc/1af7ba976ae5328d7cd78be9305ba506c2697846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134511.2/warc/40b2fa0cd5ec6fee7e54ac999782b605cc594808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134514.93/warc/6236cb158e840975a6d78ad393dcbadea264e927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134514.93/warc/bb7eaf7154ca443f1ffc19763afe044e81507b96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134620.5/warc/0f51a47a03a2b814dd73059694cbc527a653c60c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134620.5/warc/47279467462fa57bfdd9646bd9cb50ec30beab78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134620.5/warc/e31be5cec4750f416905e3c46c2cc6aeec246f1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135080.9/warc/62bfc5bdaf87b10cfa46ef754bd418c4712938a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135080.9/warc/bc805f973ec8d894aff555c68252b90a3ea69863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135396.68/warc/30fc9165365a48f34db311b590f941b75e9414fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135396.68/warc/88deed70c8e51ee03b87c5488b62640f2bf90c00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135396.68/warc/ca3a839dfa6585dff1d266f50bb25258c6c899dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135549.24/warc/817d488ee85cbc37d52177731d9264fac22901aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135549.24/warc/f7472f987fbcb5527e1775e2ac1c08212ef71eda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135558.82/warc/0705ea6c796cc9870506b75a33bad47766595dfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135558.82/warc/2d309eecec397c3f4daa8a85d1d7e288e082e9c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135558.82/warc/889fcee3ca24fd784d909ba4aacc2a44b02d5a53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135597.56/warc/aa8f69940dacb99d0079a4b9fd1ab7331ce7b81c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135597.56/warc/de3b852c40a6dc0695e0ee8fe282c58d65219976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135777.13/warc/0cac2e2d980b74d0f44246182ee3e7075d7be38d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135777.13/warc/2a94c7e4e588156eac676ccc2ae4cb78f367feae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135777.13/warc/e94374430db2fe03013df2f04aedc56e68889639.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135930.79/warc/38abc858d17a1164af2ce6dd3fbb76e83fcdc0fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135930.79/warc/8373d3ca579a20ab5a794c018c606c0978b4a509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136181.32/warc/71c53a8e221a14fd2fae028650ddf029e767378d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136181.32/warc/9061617a7966faa03f7456de7ce374d8f8742641.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136181.32/warc/d4ae869852d4a31c9e21258912c9ff6cebdd9e2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136494.66/warc/391293438efe4b5458281ed4e94241b773429792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136494.66/warc/d3697fd383c1d74ca2efca56122eafd5c7e339f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136523.61/warc/159c8474c04a27e6f37e2732ef0edb0df15041bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136523.61/warc/a3c2a152f2f2e50dfc7b7270ce04b98a67a0a793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136523.61/warc/a5a93927adae48ca1867dd3525e43bb6394f164c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136545.14/warc/c10b42f39818f5057b3f6a54f71a413cd4a05802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136545.14/warc/dabc3dd80132b0e6218c9743630ebdbf60025896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136896.39/warc/3e92039d642848c8a68840e754aa791484b93554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136896.39/warc/4691e149b58e7b3f01ac3af11420ec0282ac0e0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136896.39/warc/a828bfdccdab7ac6c18216375e3e1f728082b4a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136963.94/warc/3718026c2dfdacac20e60405178de81267422675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136963.94/warc/df367f129961b79b3875f8efe92e1f0af12a7587.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136966.6/warc/728597696f69b0bff6f54de1c732296f1bb019c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136966.6/warc/c1aa66aab540a52120059de0a1dae27617ee25f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136966.6/warc/ca45e64ebe701bf7aee672b78f7d26985dba7f99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137046.16/warc/6d19754c4a6af82cb5888c319c91e0b13b611f47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137046.16/warc/cff0da72941181fdbd640538d304df87f4188e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137056.60/warc/2b07d9ee6be885eb7ad3aa4c88bd175905e678a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137056.60/warc/386ad023c3fcb6d6055e1b92c8d173cfc6547117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137056.60/warc/5dcb272e5373ee673ca41b13d630930acaf3cbbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137108.99/warc/4bd32daad8523a34891c3f8c2544aa31dd50d0bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137108.99/warc/96a3687dfe7b4b3d14c574526bdc6b5d7f412a96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137145.1/warc/3b50264c652fbce194863d4413410d3f324ac578.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137145.1/warc/b08994588e07968ec78b47c8126b493d04d7b9e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137145.1/warc/cd90a87d1935143717050f4e3f53112c888e82e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137190.70/warc/1e66301ab3d188f7661c65cce81eb21ae5c3394a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137190.70/warc/d9433cf9c406a40855429d471ff42e3dfa7ca7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137698.52/warc/469c4d064a8f90e6db17be80e9461381ee85be91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137698.52/warc/677f260f897b9d6e4c4a060532965419ff9f8c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137698.52/warc/76f201209a250e3ed36d7437ddd3933eb06594f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137841.57/warc/24555d23d20f7a0e6976acb7cc321b94fea76a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137841.57/warc/cd5c69892dac76203bb2962f6d50ad5b3be8d305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137906.42/warc/1e7661daa57bae6a454098ef6234f28932d03821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137906.42/warc/3494105f1c1b57f00478d5f7a988e47e98ab8f1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137906.42/warc/c079cd053fad25d924088713e8e65d70d86b3c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137948.80/warc/7401132ad513a82a49a32e88dc4e58e8552d8252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137948.80/warc/d238d76dbd971728dc7f2865fb83adf34daa8319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138017.47/warc/bfcfd1b74d5f83a6fed01ab17d0d13048c4ddb07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138017.47/warc/e8c0356402e35431c9353ea505bbd19e63e3d165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138017.47/warc/f99678777c74170ae09448cc501439da26a08da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138086.23/warc/6f19c6ba8f70b7e2929df2f29b17def771701f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138086.23/warc/dcbf403a5e02a3b4ea50069ca85b5cb6c3342d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138501.67/warc/9234294269cdff241e76ee326d75e10e2ae55a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138501.67/warc/a5dd89c79a7da0333bfde06042a7a3c783813dae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138501.67/warc/db3dedab21510eeebc59c47e5fe8a714edc9d259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138770.35/warc/5cfb46a889be60b6f87c2691f7e4068f36b054c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138770.35/warc/ea289e27b46c8ed7f70c556a2d00834a671bc4b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138980.37/warc/072cbe63aafcc38629fd3fc11731cdf37438e57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138980.37/warc/6dc8913bd9f3eebd5cf65b8486de3012a2ab2cad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138980.37/warc/8da089afd28ebf7bcbda268919698144b0f793e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139314.4/warc/555dda870c1800f15e544cb0d8ec2283631a1c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139314.4/warc/8337b814944ea143707baeee561c87708f333cd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139395.59/warc/5d39b9a83112768a186e6b9fccfccf8ec32168d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139395.59/warc/d619909315c72cc3b867e2558441b4f19514e6db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139395.59/warc/ef436520de5264e8cb94a4b70b93edffa74d4ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139669.58/warc/802cfb16aed263bc51bdd2d78afd1a6748bd0efe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139669.58/warc/8e802c57eb056043684264fe0d25537287c316a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140379.3/warc/8d5b5b2805a0eeac726e63674306c7dffb613a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140379.3/warc/9e3506e58ba2dd3a81856712978e203d6e9ee916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140379.3/warc/d27b146f8dda25f9a918830c5137f13b0f4566ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140890.97/warc/4bc27e994ca0da0b5aeaada9085d1530ac4affad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140890.97/warc/920a422b4c60c873b2f81ec68bfd8b527a02bf9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140890.97/warc/f9800813db6308ec0ea6a80dee6e9e5a127fe752.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657141651.17/warc/6e06f822a74f5df140844c5c542152faad03cfe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657141651.17/warc/bb1fdbef5f8e8beac69389a6856e2595c937046d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410665301782.61/warc/4e221335d8002f9d0a24de9c341d6d9dac8f612a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410665301782.61/warc/71bd136e5de8551362222b8b920a3744e7caae3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410665301782.61/warc/fa86dabd1e35a888fd1a507061457baef74b4bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662880.10/warc/2b12f17cb053975164479264080ef57bd4564a5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662880.10/warc/67bc873d7469316088426e59fa03fb74d4739ff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662880.10/warc/79c4ce8eb481ad16f8872ae4956a06088d494d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662880.10/warc/948a8af01405de5f27a0af6b60a8740b5b42d324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662882.4/warc/0a549ece1e450dee0f31f326f17051fb4f8fe3ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662882.4/warc/129f567801602a708147c385ae9a700bb3d486ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662882.4/warc/4a32976c3483a19c0fb26a67a0cd0e882cd34a32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662882.4/warc/e854802960f69cea0546bb349cd6f2194c555b88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662910.19/warc/a7857c36453d00c9a456ae5bd7ac24e42485f6b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662910.19/warc/b025396017659a984bb98bcdc23238a74a4061ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662910.19/warc/e55a7bf46348d0b7d23b436bacc7d3e65ade0be7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662910.19/warc/f9475fa898f2aca2aa8679729dcc1cf112e7daa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663007.13/warc/117e6a603bcf7b1a6364342fb3b88efabc03af57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663007.13/warc/45daf078f7c5d5318f1ae058d861f342cd6c00fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663007.13/warc/91945507fa110d6e3df2d4b5c9d07f2af8829130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663007.13/warc/93bc220619f300e19dba7532ff5a0391470a4a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663036.27/warc/1e3280480faef91b7bacfd29b5217cad3944615d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663036.27/warc/24271c02e3f6cc84613fe0223d3852ce36663fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663036.27/warc/6b83b4c10c227d0cee30e898a145fbb04c1f4ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663036.27/warc/a87e41b399b2975883ff856bedb7ed79eb56139d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663060.18/warc/43c0e3cc075c49e0a75d6f0a249dcbe21d152fcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663060.18/warc/5f5f8b48686ffdda499b4031091437a9e1c043bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663060.18/warc/9fd1fd1644c6bfb8e868d3e9ff1c8df5eed524de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663060.18/warc/bf9d4ee08989507cea168a7850931985c0224881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663135.34/warc/29f52180aea16d8a2e1c9228a14371be8530f241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663135.34/warc/961eed81694c4b0a74d47ef56d233caa6ede5bc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663135.34/warc/bcc4bfc35fb954caf07fa6af54a886b720727c3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663135.34/warc/f918bb769a5e95ec86959dd0183d373427e67d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663167.8/warc/03df8d7dcdb752c3627d63f3464af1368a8adcf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663167.8/warc/0abb2d1b3d92be0938246f08b5d0d101acab09a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663167.8/warc/94fa2de8184f036a88354549f2358b8e5df5cd27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663167.8/warc/df74b58c95306239bb9e35c555de9f5cb726aa86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663218.28/warc/47cdde28a90e38ea8d47247bb4c59ebc46e56efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663218.28/warc/75958164fc60f426b2b6fdfc9dbb61ee74163ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663218.28/warc/8efcec34fd84eeecdbc135cf4cc3e83d82941434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663218.28/warc/ad229897067fb25e61d5fcb0c872cb8a32387004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663266.24/warc/f9142e8d44ed6e94299f40f4de3bb5555f771afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663359.6/warc/1ff569e16bc74f4e405b2badda0dcc18109e9dc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663359.6/warc/33c1d050959e5aee3d980d075a3bf85cc2fe5c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663359.6/warc/93ee92debadb515b84542bfb2993181ebf5aa0a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663359.6/warc/b6ff6d02355d65f7b0e65f1748965709c696a16c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663365.9/warc/22c4b33d79f71f9ecde530aa1978e743e41e8011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663365.9/warc/25c26c6ae3907248f0928247d62e2c5886758afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663365.9/warc/494c5aa18b8310d76e66fbe77fd35c0ea920cec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663365.9/warc/5fe432632abcdcdf97d1f4d8f0aff3ca2fe76ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663372.35/warc/0315be229b2f32cc72e33c8ad1ec4f6178dafec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663372.35/warc/0418931b19234960663fb4bdd70f2b79eaa9e8c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663372.35/warc/30ec63045caf7f3b231bb1e33f999a9736f94f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663372.35/warc/ded2e06488f39187fa9284efe3be99aa07b92da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.12/warc/4d68fa956526d1aa9fb837d451967636636870c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.12/warc/5da6b9e5769ee191236017f6ea9ca0e4c7ba0897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.12/warc/9b6933f46375c502e542ce45aa8bbcf8498acd13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.12/warc/ef003f2c5988f911f55ec901d107c2582e3936f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.16/warc/5d850a297c0a5deafeaa7da24da843797d3010b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.16/warc/ac63b0f2a77ab3c07390d6c4d415a72b0a469ce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.16/warc/c16d5f589d400c9a9a3eb7947015578a835d0abd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.16/warc/cd9954a1ca5a7e1d10c9573bc6db7a9de8003c2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663460.43/warc/364ec1fc88e9c50c7189bf8d58b67b07488b02c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663460.43/warc/93bdcf6ab338073774bb07aea62183ad7317fd95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663460.43/warc/a4193ec9babd6d068e040eaba3545624859d727e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663460.43/warc/e4c2ca2c543c2b389f7d7c57f807fd25494c758d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663467.44/warc/0c7505086074dc2a223074e5e83105efd02b8fc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663467.44/warc/4b9bb9787dff43b0cc2012a2b3e8c831371ffbe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663467.44/warc/a5d46423cd9e86363b5af965ee5f6b0cb5b91d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663467.44/warc/c3d7cdebae47e1bb6c8358051cfe84d4d1fe17e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663551.47/warc/1921d7d0ac2b9bbbde3fdf98012602b2de668acd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663551.47/warc/a2c91777940f92fcd58d1699367e2c4d5a36a516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663551.47/warc/b22058a1bb238a19ba9941802fc6a819d4164fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663551.47/warc/d5c7ea9ad8746fdf7d9b78e8d0032d3aec61cf12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663587.40/warc/856ae3fec50421741b8b7b8d2e2c6bd0224eca48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663587.40/warc/a4ee154ffd91f84e3b68c2034ec074a49df9ec33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663587.40/warc/cc1855d7ba2d1374f64441e656e1552a41aa7e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663587.40/warc/e0902b5cadffc842f635122a3c9cfc3d14e8a79c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663611.15/warc/1cca453cb84f2a291b13c916c2d9dd76efe385d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663611.15/warc/2c7fb3a75a839c2aa035c0b788276e8c8e2848a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663611.15/warc/574284c0102d40b4f691f7ab47be6b7a9f8145f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663611.15/warc/cbc0ac6c6dfa6a4838ea2e546e1878d3daa1881d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663612.31/warc/453c5ed46bedb1a17b9d96fa2f2f28245cc00120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663612.31/warc/d5bd0a7b384b6007f85343df60c3b6ac6f499de0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663612.31/warc/dfcccaa1264f1adf65bdc38d54cd7884d4bbfc03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663612.31/warc/f6302290914b58e8ded144bdf67dbd8a78c249dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663637.20/warc/425b2ee015daa333a4218223dcbd7e1cead42031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663637.20/warc/6c1aa63ee88fcca6049601f833426c98ba480359.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663637.20/warc/b24a002be2ae7f7a6eae8efa3bf02acd2f227a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663637.20/warc/c359d4020239e124c4099b0e1f43dc23828b0c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663711.39/warc/03f72d63b9937fce099712cb490968520c4c65cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663711.39/warc/35d37b025835185a763bbd936de276a88688070d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663711.39/warc/4b703bf23116a4455cd202ed6477ff5db9675c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663711.39/warc/b74e6eaee57c886d8507c871036a8e6d4525359e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663718.7/warc/3b15a170c18b957994733f7bfce962444c5a8a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663718.7/warc/4742b6e31362f98a9abc22d2257812bcebfa171d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663718.7/warc/acacf05e03ca5b6eb70719ff01ac811911bcb0a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663718.7/warc/dfa85961a82e77b959896554dc36152d08655bd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663739.33/warc/65da1ae26c94e072597d68925cbe39a2c0f4b9ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663739.33/warc/6ad320a04d518c11c01fb95b3a54f297b44d3327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663739.33/warc/7ccbca3656174772e8bfe8f5619a5d808d8466dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663739.33/warc/f5a6305b1ce114da8d05bf42129853b140e0fcd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663743.38/warc/65c3a76026757e00427c35120c1646a0e834c786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663743.38/warc/6955767cee9332f87a5c54e4e64a7b53cb462123.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663743.38/warc/695754b3369c2b5f757402e6f4b48b8846d00651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663743.38/warc/f87ac50f0008301b999d13003999d0444aba07fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663754.0/warc/35f5cfb102b5de5c1d2cb94107a25ec3bd4c8463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663754.0/warc/8f09c1b78e061b034c3e2f219ce4c6a9ef482f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663754.0/warc/c2daf295df0d24c4e0592b907adc443935c63941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663754.0/warc/e63650b653b799cdd6e61d0c0c022b313993ebdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/17c4492eb437e735c1b3c0a45454ce263d9173d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1927792eb82ec72d4e8f3a2abe8a73e198d6823b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1e76151b3cf30171fc0603aee1b4f01121c7d009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1f8621708d3e2b844718cb2c5152f8e46159f21f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2264485c732aac13c59b989bd68a534b16967f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/24f0f1bee90e236358737e462fad78d6e5cec37f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2663989d4a674fd87fda731c6d2736a1d339e7a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/268f9fb8a5fbf57171c10f98cfcc753c4cdfe8eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/269905fdfd007524479d4a3aefa2911aab59194f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/28b161b3f7b8f1310414df2b8c15aeaf66413e68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2a2a906d2b353ce65e4af18e891fc3c3048dec88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2a329af1cecdf56fcc44aedf985399378ee97ba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2aee3c279a1b6f678105157f5459a0972220170c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2b2a0718cf6901e3087deb54043742d00a8d2f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2b63e6da9ddd7aaaefe12382456336bc8682a0ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2ca2f9448c9ed913ab2d0e060e4dd24937c103b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2d706cdf93311d2dc6f3b0f7bc042caed49b6a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/35f7122fd804719ca1587b3ff3cf756c6502a7c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/39f9bff3e38141f80e89f44fef91066ff93c7937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/3c56fff984830177fe4e6e7d84619368af0c508c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/3d98dea0134d024c3c1eee15ddd9de86943b133e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/407fdb5878b3ff0342328705a3e9f5cdc5b55323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/471a11d220ad4d9bc60e32ae1ac70f0afd88f3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/48cbfe503fe26c610806299eb0d3340ac0d7d728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/49022b8323df954a81e70a6a37c1b3af8a365956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/4b8c6a3b2b99b36c90f25ae7e3b5ac6db94fa59c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/4bafeb7bf640a60037aeccd354d5aa5942ee6ba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/4c59c0d1e6dd571885b50ec39e8d0605b2495b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/4d243ec6d1e71533b8db262a32f2085795d75dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/4fe9491ee537ff35b5473a1b7218d2fb63382f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/53571da664d482f426f3570513ac2a6f2746cef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/54ee32c65f2996d98a5be06654bc3b928436978a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/55d66f6714b916357f5339e6b8117630b8e97935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/5664e30af8b1e0320c4a44430c2044a94ba12a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/58924a921c13ff742a1e7c9d453c091f3025d74f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/59c42341b23e67fdad9fb3d7b684b93157f94e88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/5b32964ca3e1df26c1225aa4614b6f96f1932039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/5d75b0005e870604af19cc143be5554f959531a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/61aa1ef1af3363033c8047145399a2231cc0625e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/622abed095487a7aaa0e4030c065e8d6e554ee5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6427deaf7f85dc0eee031ea77e69aaf12c26a7ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/668839477d44e86aa2df50d78d71cb18f70a33bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/66a43d715b9b2df21444423def0f687585d41218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/68995d32f8e6f3116917563d5b57a68ceeea946d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/69805f316991a63d8f927a2d0b879b8f828d7193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6a106dc2dec22a6c39f651f062e85ea0edca76ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6aa56b8508c15d42c7b5506105102646f0fd41a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6b7ec067dbb5618d50315fca3a1a1bd8c858de8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6cb0f305dc88777f57227dee1b5c605972558e6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6cc2c3baf86fedc38e86f109bfb70f49ccf3c390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/72b5681a5c5ab60f3b2ee53bd6aad5a23f7603dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/75c07c4e0447038fc2dc2a0d6d878cd8140979bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/7d7d6707efdbeb35dfdda06e9c614f546b0f2a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/7f476a25bf2d06ea0c7e7df414219046e819ff63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8114a1ffa74eae86fa0bd97a0c8f66050c8f19ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8128a469cfc5d15459012b3f809d27bfc65c348c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/815d37b0191dc7c58c405c0a55c2cf79f7109547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/83298241e8ea9010e9a7daaaba6471534cdb0083.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/841c261cd8d18d486d4a1f0eb1b1fc66706a3892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8737b08f3fd3b8c409845d74553675dcea772c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8ac16690a84ea1a36383590957ec24b3da165522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8ad13fec2c88568807d3cc5faf21557bea82f28c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8ae61fe35b0d5cd31d692bac123f4ab8c7043137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8b213033322c0f71f1676a286a073ad62778404d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8d4abec8880b53ee0a64725b4d22331328d3ea8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/910ac374ac816e9b00c86a6147923348e00708ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/91568bec38a419a4013345648fe1c9318a993d41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/938dc9e5addbc0020de47242a515af7151935c35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/98170280bbf212d418ff18e12cb71de28000ee1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/988c358c9fa70b26faca446c58922170a8411963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/9e4d2e024bc3bf24fb171264619c8771094172ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/a5857593d016d7ee7bb7212ae5cc9682762b6eeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/aace20df63ba8ee36eada6b52286c789afec186d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ab0af9c583617a36061c10aaaf2996ccc7fd17ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/aba72a5c19fcae9a39c21e5cb1c00d9e4322e123.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ad0935a362a9098ac9bcce26cff8215823ef6e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ae5ac17ab19c3cbd4d7fa3731e4e0c47a08f8a3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ae943bcf1350bb614ed59275ac5d1040f0b14f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/aff50c8f98ad2a4553c121a31619298f5794d100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/b2c9fb9f52cee7578fa7e1fa7066b3f738e21395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/b35cf4ad8f390dac6ae9f8bc207e64049c069bb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/b57c6fa416eb9f8942c7173a18a9c965aad105d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ba2d440c774a8ae184ebfa109405de77f99d480e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/bd0e2aaecbe69dc1cbd8b84e5e12dd1d41815ccd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/be49313df1f5899bb914c67b19a2fb358fcc5dcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/c0ecc1b296a3046f700f697ee206310419d809ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/c3fc729b0d38158dafc42e5d27e6df13ff18764e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/c5a6b3b054c37b575f3734493234233c6c17aa8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/c5d44ae880fade5133b25a11982cdef305ba77e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/c681e557362f2c5f1143737ae9831956f070d9c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ca1f17da70b395f7b31f7c9596743011f99a40e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ca7140bc273641e3e489dd96df500a3a464ffc23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/cbacbbc4c3e3e7a57bb207353d98804e6cdcfd36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/cbdaaa15aeb3f4fa8b452cc8a59fecf2a62ad42a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/cee076de996aee076c3148907fb6a0ecd67566d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/d1e9dab63aa37fd89d4c5dd65922f41addecc82b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/d82b70c1e6ef442efd420eab1927630a1ce904a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/dc3b4743c194a495bfacf63b524350b5f21d9c5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ddd7f01b6d12ad609cc2957d891374b7e27393b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/e02f8c5ead223fa4beef0db58b641d7aa943a424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/e19e838220a182afed247cf1a0c3561b638c9f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/e5a14e2efd62217bb3144a0a34be506ff5152c51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ea62458c6b3e7e7d7024971f2434fb7cee4a7942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/eae170149fd8667f9b18607a720c171a629fcd13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ed2b074e56ea0840600e30a57b0c0b74aee11690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f75068845c8c32144465e0f130fe14bcca949dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f7d704fc769ed08dea02d5066c70b124ef2bf940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f8285ffb90ef0b6eb4884b3e8a5971cb726297c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f861ec0be7988c043ffdbdc12613c907939606c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f8cfffb70923b69f7cab2a4c06a1719e81c6d78f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f9d178d384f0a41bc0c77d19f93d134bd2f936e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/fad76d756e87ab9ab24b2c89ab40083002417c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/fb188f2b42bd26ca3fb000db20ea339471de111b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/fd432cf6b52e977ab30b4212d536843025993a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/fde57dacecaea7b9c05210ba987324690efc99af.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2014-41/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2014-41/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2014-41/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2014-42.yaml b/configs/cccc/dedupe-url/CC-MAIN-2014-42.yaml new file mode 100644 index 00000000..f1a2d579 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2014-42.yaml @@ -0,0 +1,618 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/00383f6644591eb0f141cc1a2e0daf59063db031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/02c4908dea8566051c2d589c053e4de73c86121b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/031663dccbc175f518a22aace7e804f106f207de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/042260e59f7707854dd8a43e820793b2d5c28c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/043aa39674bdd810bc208679bbefec60b5ff262e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/051a7a3c8ae27679f46fcd648c108799c9358396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/0a7474b9848daf435d6660b539a915fbe87dae75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/0b3218215cc0d5e6c94fce27fec50323f94efb39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/10ffe3152a1100343b17aa72df5b285b3477452b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507441801.11/warc/b4718135a15b2518f1f001746e62536288ea2f1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507441801.11/warc/da456a8b385bd6434dbbff666125b8ffd7972d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507441801.11/warc/e123ed048f9593cc2daa69f817c6cda60718a8d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442288.9/warc/94629f3fe8943ce464c09861f311ed60d1813011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442288.9/warc/a803d3d1d8664d02d614cb580304da56efdbe156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442288.9/warc/df383a2a89d304e59575c6a7e7a33943a9beafd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442420.22/warc/7aa5e55f6f80313e6140c993a1754f6c0c0f2d2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442420.22/warc/90716faf413ba38e8253731b94537927462ab4e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442420.22/warc/f3235b9abc051018107c10af861ad4fc8476956c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442497.30/warc/0bd3389c620695a9d4887e67a45b7768edb9dfca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442497.30/warc/1985cc66ed08dbfbf9d56bd620ab0e5325696022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442497.30/warc/d4eaf9a1d42aeb3d070b61faa79d83f612f73857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442900.2/warc/07ab2979df5f506ae6b8c3819b7345ac6a891b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442900.2/warc/f268fd71ae5081bfc6b72e4c3697de61a6143571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442900.2/warc/f8904143eb83e7ab0fba3b87dd4f51382996f6c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443062.21/warc/6970ae59833879af16ed214325215a414ccbcd15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443062.21/warc/9c7117a9329f97a3321781b939b2d753f43c1347.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443280.18/warc/36d4bb36e60ec3ff4e1cdc625f46af7efc97238b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443280.18/warc/70d92d272e125871567e4614566f2325bd0ac672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443280.18/warc/b235d25def52b2025458941c1d4f60d6a2b11fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443438.42/warc/dc69680fc667f576e0eca6f45bdba31457b1d89d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443438.42/warc/e75bd86a7b8ebffa3152fd04f2d77edad93b7b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443438.42/warc/ffa00dadb492442424a488a2af7d70d159f273aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443451.12/warc/0adfba1928bae9f5eaae6e0f0e595233f9c8e963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443451.12/warc/67cc2748d98fb106a306a86a84b2ca7f224558dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443451.12/warc/d6e78e5a2279183ab501f9b20a969af8c51bfbdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443598.37/warc/29bfb2b294c5dff5b4a9ddd7d0b4575a6b1da602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443598.37/warc/2a652ecadd47c583379b2194c573a58ab0f90d24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443598.37/warc/5b59d06b8cb9d46231d3d41e2461cbc98853e1b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443869.1/warc/12f2e1feceed30b85420c7f40b93889ecc6d6b2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443869.1/warc/ff93ddcf86b75ff799074d568dbaaab4d2484084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443883.29/warc/4ef0161c199230d94106d8c7fd20adac765c8911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443883.29/warc/6937e37c25a366ff7e329919b389f7fd4e70871b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443883.29/warc/c438974416658a0d22df86aa6c8e6af8df329326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443901.32/warc/2e8a8aeff60296ac6d9856083692dacec24a9f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443901.32/warc/5b3b76f82444fc3b54440726ff8372416b6bfb76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443901.32/warc/820a765d475ce7a191ddea47714b3a3c7f4d4270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444209.14/warc/33e98db944692b6d030629cb5fd2e7fb721145a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444209.14/warc/5672fdf111645de127cceb01d631deada468b086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444209.14/warc/9046549d5e21431ed8903ab14bc2ea9ae149deca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444312.8/warc/04a8b60c56b8bedcbc55c5cdabdf2a9d84429791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444312.8/warc/36fcfd3b8a549d1e6ec2d125e1fbe47321469942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444312.8/warc/ff2064b718b6ba4f2d787ecde2cbecf41652115c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444339.31/warc/26fee23923969c287847c1027f0a45575b0eb35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444339.31/warc/ed912e546868a2feaf51b915b4c6bdb01ed833c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444385.33/warc/14379a01f7880e31f94eafc90e291a24610c0f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444385.33/warc/7b262c0e5610c0cd911c6b37ece2fa13596de983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444385.33/warc/f72080724e1537d93ad9d8b6b6c4379685408d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444465.10/warc/31d925ac201841d1fc408a2583d8842afabb342d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444465.10/warc/d32783a412cbf5f1ed89c21683c58ce154c0f022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444465.10/warc/d5463c2782a674f305a350d81619123e416a2f5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444493.40/warc/07e23c75c1aee48f342f3e6630b6e56de69ec81e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444493.40/warc/17f2b7f2eade4a1e3330596eb510e0a6bc348be0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444493.40/warc/a077df36e6a5690051a0fd6176ae1bf841edadea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444582.16/warc/3138f03a8b6c28219fea592394ff1a2f5c98c8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444582.16/warc/3d22877a3c51beb0a93d8ab2522fc67f1be0cf8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444582.16/warc/c9db86e069e9b10aa62d9efb52a784f28781c309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444657.46/warc/1a8e20662f5313452579fd52b3410a85f794950b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444657.46/warc/aaad847af54b9c922364153e96b5e5d320e3019c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444657.46/warc/c5f27166254176d6807c97b6a8fa71c5391fdea1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444774.49/warc/6703ee86b0082836bc5fe5ea2f3d20619e3c6c53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444774.49/warc/b22b0788f808cebba30996b0f348d7b4eaf94d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444829.13/warc/069c88501d706c2bd91faaa297620f25bdd140ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444829.13/warc/0ab448b13f6088009d1f5d48068e99705de1a9d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444829.13/warc/8fd49c966d68c7c9ac2f76712536e4306fe963dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445159.36/warc/3b3efedd5ac375f26b8c36e867deff0976f4d56d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445159.36/warc/50c033056de2397287a81edf298d94f21274615b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445159.36/warc/ed3e6a5d32736c4587348ff937b21ed62b9ea9ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445190.43/warc/4c9874d5d4dce1050a1a0256ac900ef4c446b337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445190.43/warc/959c9572244cc39f9116462a6f3eafa3b7b7a5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445190.43/warc/fbe5fbcc9b274f1d282612e0592991263835ac32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445299.20/warc/26de752272e28df6c6412814d27bfc5c0a5d5462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445299.20/warc/64de09309e5b530fea5258bf399df84faec4ba33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445299.20/warc/afc906a195968c7e37ba026248d652020c9895f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445886.27/warc/96ebc982f38e4578bf4012a44eb4ac98aae4d44a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445886.27/warc/e66512002f71994fa23a84b317c761dbcbb2a435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446231.28/warc/2468699e99575129f4886bbfdcdb44f93483177e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446231.28/warc/2e27864e802fac39cace25710d432a0cf1d5d90f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446231.28/warc/f6e26e3e25c10407a479971e928b966cb5351637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446323.3/warc/b070eda75319ca8374c64a0587472468034cbc63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446323.3/warc/b85ce5d028aa322f2de4d7390e0b1c72c8e6a035.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446323.3/warc/ba015ea114139eb8394d4c3a34ae2fcaa77c30a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446525.24/warc/543653fe07997ce2767451708dcf69b7a250a689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446525.24/warc/807961ecdad66f6e3b397eb00dd25df14791bd4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446525.24/warc/9d41b506b15c5175a3df11799a4d4b249cfd5379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446943.4/warc/0186296bfa45c81f5809a5b19604591709ce3952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446943.4/warc/b3034a86bfe9cf64f447d0f3a134edb75cf50f70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446943.4/warc/d5d9069e57273158785ec9985242492b10bbc117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447020.15/warc/4ed8bf0d71d516b436391032cc3c40c5febfe522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447020.15/warc/9f1a4961fcd53c08701491510c8eae8670320f9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447421.45/warc/6531af2823267295b572de7d9dac177450e4a4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447421.45/warc/9771cc93092c30e62815a91d83404a7336d7c473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447421.45/warc/a382bd132641a60751bd9e8620aa34cffab6927b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447657.38/warc/77f8087fc8aec354ad8b2840281cee8ba7ddd98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447657.38/warc/f0112b20ff4fb16b2410e4a271323f5057007c22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447657.38/warc/f51df20485567037a512ba7a8e4b54bb70cf5534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447660.26/warc/1c371f14dfd59000ddeda4729bd46118f344efcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447660.26/warc/28500886242addba92c55f8410c979121b9da5b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447660.26/warc/399b41922e8e869a75c3ce9ef369563015fec317.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447819.35/warc/0b6ea9a9ab74ad01d62618587c5600899bb2e88c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447819.35/warc/2350a7e49d4f771309332829a87586b899bb285b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447819.35/warc/73658c8887840add301e73ecdab7dd373966e851.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448169.17/warc/4ad322850c5be82678572d1efcd19315de8e178e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448169.17/warc/bd631fd0c4a0b7ae67b1aa4914a9376041da52db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448218.19/warc/0891de2e007594616610c66f3c4a37e03dcab4a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448218.19/warc/08cfcfef263e54d66413689f009219bd97f33ef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448218.19/warc/7084e3d9ee90e3a891aa50ab05b4bb9a1814e95c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448896.34/warc/3ccdb5837c57a4fe6de419b775e6b73cee30b12f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448896.34/warc/8fd1c8a4af03578e00c8b4afdb439920aede2dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448896.34/warc/d03033adce071ac62150e637e765efc90c69e56e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449153.0/warc/0a2322086df366afcd4048e2bb44c6144fcc1915.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449153.0/warc/595b8d5ea0c6fdc726378691ce3fbebf6f4129fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449153.0/warc/a731c2d4acdfe524b4b1c6c77dff8345d8de7d38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449615.41/warc/21904a2e4484bf045ef8827075c3f7f48a9359e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449615.41/warc/60b69832bac7d3e978d73a2249b7525fc7b37ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449615.41/warc/64d35838c0b8ee28da2882e2ce33c55d49900f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449999.25/warc/0ba01d89ae0ef106e16bac3832273c5e7f905733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449999.25/warc/41ce0dae3da88ca8ddfedacd0eaae52feb1cf289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449999.25/warc/d6a6b7eca63b558992c1f42d5507caf3556c3b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450097.39/warc/40a80ee842ad66fde7ea547babe280c7db4e4a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450097.39/warc/e4b48d364d3b6fbc6a9bae477b9a42e6d786f30f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450252.23/warc/466a8d74af9a265a3130e48b490806fbb7982472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450252.23/warc/5ba2e0a4297fa3a4cc676c477dea165fe14baaa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450252.23/warc/91a01427727aeb53342b3b656c666bd630ed635d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450767.7/warc/d1c5187afedeab4ed2bb78d2133b15d96e0cd6e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450767.7/warc/ecedfbeb80f73eb9d8550a3a6689a7d4230509c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450767.7/warc/f430961f667496bf1e884216c7e8683cb60b89d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452492.48/warc/454e4ef35a2111f6cfecfcac1a6ae6a4344647fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452492.48/warc/b2f5400976955b78fd59b640c117804e4aca4945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452492.48/warc/bf16a0611382eef65bf87bfd14bf6fb5a9eeab88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452681.5/warc/7167f5036eb2509ff7bb8454ba505ffbfcc591be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452681.5/warc/b77b5b89f873e1e3fe89890d8b508056aee3dfed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452681.5/warc/d575e5d9a862ee3edb5641e64c9953205d3fe8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507454577.44/warc/04888415099c8adfcba15bbbee4ab21bf78a74c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507454577.44/warc/73accc203c71dd6d97fa110b56852c3cdf403fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507457324.47/warc/323a93caa8067e338ed4cdfbba07a40be916576e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507457324.47/warc/567a0429364fbf0eef05509eccba7b8af2244b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507457324.47/warc/a6e2ae8039b2a2062eec0450e37d87b58c4a16a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413510834349.43/warc/4830338ad8ed0f71a99c27ddbfe7a790735b8a94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413510834349.43/warc/519cb49d76a3e4afaa3b5bdd9d58b4127029466f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413510834349.43/warc/d584a4e7a0585f1a1c4a9031c18cb39674d8c289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558065752.33/warc/59bbf29f5298d80556798054f234c95217b0b3ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558065752.33/warc/6d591c3291d2edcf1ac373ec81482fe14dad64ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558065752.33/warc/89296b21b88bd9ade10b8fa68b7fd6d2bbf85253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066265.80/warc/73d2fa303bc9f09db7176b99185eb0510e0cd87c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066265.80/warc/c5ef00329480c2c5e81cd5f59b58f2bf58d5854d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066265.80/warc/e94d7eb36a138a118ae15545739eaaab87646eb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066290.7/warc/4f9a530d6535add741bc63eda54b42db3f921992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066290.7/warc/98a814ac33d76ad5a83ca29ab9dd44677cfd6195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066650.89/warc/8b9a27779c0172c65d39098c85bde4a2d19c754b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066650.89/warc/9fcb268a886e013abf54ba042338a72a8d523f7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066650.89/warc/ed4ee9b2d24d9bcafb0409b9ce5a5c31793925de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.17/warc/17d790d5abc6449a83d412ab89083e940f0e28b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.17/warc/2bfcf3ae0c39f61b5395adbd0d9d35a80b595dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.17/warc/67834c0105716d56ab6c2d743288de5ec8cf39ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.4/warc/7f75929524ded508dd784ac7e2ded78e5d2eaa1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.4/warc/87b8362d5abe69c78f21da02e9c3e80f6251a6cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.4/warc/9c1f7161eb1374a5c863d03709dce0d68f9e188a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066932.32/warc/4859bcd95d9c3c7ce0ce19845004bf824dd5f980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066932.32/warc/cab744ac18dad26bce8c51825b53f4cfb87f35d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066932.32/warc/f2763d23f1ca9aa58aaca100da76cda672c77869.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067075.78/warc/9f7516cb33e07c030a26b1a8b863592a7ed72d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067075.78/warc/cba6665029dc16bc63afbd45e6da556afc61c996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067077.47/warc/21b11ddfe4f77a3a9bd0d020a1a7cb54dd4faecf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067077.47/warc/657408ff9fae483616bb41a78c64f39904236e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067077.47/warc/da05d3e91060a23970ffa7e1d7dd32061e22472a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067100.54/warc/5da363bfa8e5c27df052d8f272e8585093b1a203.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067100.54/warc/7be0d9a5aec882073ba88a6cb2f928f14aa7477f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067100.54/warc/add0d40db43b643958c3bb5edc613a4ec6dcab0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067114.72/warc/6786ea3131518bd547f3299595c74f0583ad730b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067114.72/warc/99da63d62d3f33dfb9ab3642fed65ca9ec9a99b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067114.72/warc/db154aa185c0625ee223670ed18f7c85212129cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067214.90/warc/17f75aeace45e88885fa67aef09db45256afaf30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067214.90/warc/37540e4a5c8832a0d529745d76ce0299354ec526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067214.90/warc/4622d9447feecca3dc15ba9908734934afc6b5a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067467.26/warc/30b1a50d53cd858cf3a6dd68486422fb95490ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067467.26/warc/45bb5d63b38d927a2699d38dbc523021c41b052d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067467.26/warc/9e794cae36528762c1caad115ed781b81fbb04e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067496.61/warc/f2a39857acddb5e9c703a5e7f64790ed870d3cf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067496.61/warc/f8a7d1c68527b17e05e8799041dea1b04034ad88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067609.27/warc/25befca56aef1578a6929a355d55f0a3eeeebaec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067609.27/warc/bf9abd30b0c1494b6b5e69b4785c286a56388e60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067609.27/warc/f12abc18a77e8c7e42a51e14b0e3ecfe839065d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067648.77/warc/04bafdcb7e4119fd6d6a05d3b39dffda4e9a7538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067648.77/warc/37c2362713d7007eea4448673ac6809f84dd3f02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067648.77/warc/641f0657b4e0140457ca6b143b2a1baddf878381.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067768.10/warc/46ed747d639193fb8e51d631b62a4280beaaf4d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067768.10/warc/9fd89f1f5dff09f36bf5a43a4956fdf23eda55f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067768.10/warc/ce083dddbb6527c2d4b0d3ff460a5c7f98d004d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645329.44/warc/1721a2473bd80517197326d353e98bef2b74ad11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645329.44/warc/b22b78fb25699a7b3e99d8884ae423ec71a59a16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645432.37/warc/095da43cb982de9882859c9f562596e35db03fec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645432.37/warc/474d5858246fce9b00897f909c84b2fdd9ab8cb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645573.31/warc/20ebc3ceab12f254b37e768f66a4c1c0bbfca2c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645573.31/warc/6411abd1f831bb96c872e0759774222661c0dfaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645676.15/warc/9216e9676fa4d4017cec82e0d44701fe0d5da4ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645676.15/warc/c0b2dfe075ba014ed39d94b31873bb9389a04256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645676.15/warc/c8b8effeb01157c9cf9fae14fa558274c16e0ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645845.57/warc/b1d1d1751b61c77e110fa8c3d25c8937e194a541.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645845.57/warc/c9bf07fadaf6073bad15e4521498d87537b2a559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645866.1/warc/05703d25fecf0bd912245c5bc77592a5d0157898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645866.1/warc/782e4bcd6eb63f5ba31f2140e707b6cfba10e751.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645898.42/warc/2bac80da6e1e3259ea09be16f43c365e330aeb2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645898.42/warc/79f522eaebe99812671148b80dcda8afcdfcce9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645920.6/warc/94e327cf27d73654efa5da8bc092c65d293673e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645920.6/warc/f5f77615a8577a91d998b5cc06f219a5d68d718b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645920.6/warc/fc4eb57839425d6a1b92a9c115bb1009b7e4b324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646008.13/warc/f9c33a97f5275f887e5ad0dbdb4ef3b0263ba42b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646008.13/warc/fa90a76773e81cf621eba1eb8d8d806372070682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646180.24/warc/2af4315e4b120a829142ecbb2c7bea62f82e2196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646180.24/warc/2c56f2649e8f73f0a13abe8a0dc613a9b41ef7e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646209.30/warc/7984b7d7389b28771ebd7b9cd30345251d09d924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646209.30/warc/c49d60054428ff5f8fa659ebc02de42b514442a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646269.50/warc/0fdda3a818269032b30a578f178385a33f00f50b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646269.50/warc/18104a1290acdeb90027c15a2ce376e528297c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646269.50/warc/8024fa5eb963b0e92051c4028beeb1a580320182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646351.12/warc/4fa6ea61f35d88528a56b749193a1997f79c4a3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646351.12/warc/f7c51d46fcf9072ff5dbf5f68778f6087298128f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646352.2/warc/04587bf6675f6dd85ec5c9692315a40a85366b0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646352.2/warc/44f9b5de862adb3b5d1e5073b06f933996804af7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646425.54/warc/19834b48f9553845201fd86aa3b92e8f74cf7acd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646425.54/warc/7e3e14c862ee471c8ecdcf878b36a3eed744c414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646484.16/warc/755d030d8e7b5ff5196c2449543f33521cde00ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646484.16/warc/d6bf4815905899f1ab9c2847f09287a62bc759c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646518.17/warc/2a829cba86572fe94e19ab6d15d778b1ee0ce82a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646518.17/warc/4d5102ab23129a428b44a709ce51d602c219463d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646518.17/warc/a946b85e16a0ab831815316e0a57e48d2d221659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646519.4/warc/3eea5cb1da25ce702526c83f4fbdda43efe79db0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646519.4/warc/8eb77904df23d0884f25aca8d2daa5077b8bc3e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646554.36/warc/089efe7ccb64b663f0df6d66622ee83baaeac1bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646554.36/warc/e3cb9f0a50fa05cd7604032e213809e1b25da879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646849.21/warc/46abc5621f94fad6da78cd66e38fd44aacb45346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646849.21/warc/4e6d980597803ca7e257a34403e5ff246d5380bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646849.7/warc/0095e72a5fdc2e4fc9fac9405391c39b649699c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646849.7/warc/31e72f8a5e8f59fa0b8c4ac8e7434b7d021ee9bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646849.7/warc/7f5869e8e0cd126f09263656936300b0e360c701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646944.14/warc/61906bc8df5a10dd7830a24d5167be91c05f9d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646944.14/warc/bc0eb56684a4dfd6334add41a404d5f79c65273a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647626.5/warc/b46c47e4013f8c6b7a3a718130bd0f20998c4288.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647626.5/warc/dac281482ed36a6cfeceb7d96e9c6715842be96c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647629.9/warc/b3e5550566d79f4cbbefbc981aba2dc89b733734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647629.9/warc/d5771460950a57cd1f2a70810d9c66b29189a19b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647778.29/warc/677cc11942c864b1eea0e3506131b6bb3a7aa7c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647778.29/warc/8f2abee9e551fda58bd9422eef3ee001c9d2c71c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647778.29/warc/b2e04fe700ce82e2d88dc9875afd9df06e35df4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647784.27/warc/2345a2cf4325aeac5c7c84b94f697b7514cb1338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647784.27/warc/a4091a293915acd903c1dad2aa369c3c834ed2bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647865.10/warc/be6440b34701d6cee5f4b9d44e3d7fb8780d8c3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647865.10/warc/e3b87002b99d430c8c9cd50f06514743a860954b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647884.33/warc/1450b973d08ce7c9367c1be03f552337f19aa573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647884.33/warc/a126e1943ee506ba7507c2226318ec44e2dad0f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647914.3/warc/b719295546c9839de0e29ccdda8350f3b7b05a41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647914.3/warc/cd89297a58c022b77bbf9600c7ff1fc745bd986c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648008.18/warc/4de1c77158b8ebad1491f19cb39681ccb9464daa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648008.18/warc/b00f388ce4a38d7d0cf8e0662e8f6fd4c57e5441.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648008.18/warc/f3c419c656247a3cd89bc4b904650e1c1b2ef954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648146.0/warc/5583cca218d0a8c3113a89d8ef65487ed1fac298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648146.0/warc/cb1ed0b80520792dbde759ba968820b7ceb2bb3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648146.28/warc/4729551694919fd17806bd7d59caac6c1b5ece99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648146.28/warc/b062e1c2783fa55f15890c26ed7b05310805fb47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648148.32/warc/187bfc0401332d92351df8e6dcf53feef9623b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648148.32/warc/d1a3d924652120b9377196ac8fa8064074fb7921.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648155.19/warc/2a20bd0c8ba0576e8a5b0c55c11f434ccc361dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648155.19/warc/d02eaf762ba3fcf12a591bb416899a9cc84e44c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648155.19/warc/f3578172105276c8f0dadc61c3cbf0d23beeb6d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648250.59/warc/7641975c6f4205891ff7640afd8b79a790154fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648250.59/warc/b04c230ff5f70d795ad01dae8564ed118e586585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648297.22/warc/6405d3091d355add8fbd9b361f14798c476399b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648297.22/warc/cf4d2ea9235fe26bf91a72919eb161685fc9b1dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648384.20/warc/74bfd0d4edfcbec034fd70ae1581355c9578af9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648384.20/warc/b49163359a35c85afe17d416e2baeeef52a12558.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648438.26/warc/24a3da9bd0429a52b381a986d7ba6a62ac2c746b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648438.26/warc/ee247591116e781c479634a46e627e4bd98f3001.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648438.26/warc/f92fd0f68469afd697fa712e475328df7953311b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648706.40/warc/3533be1ab5999db1e8a9e89a25f1c31d82b47945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648706.40/warc/ea81586e2868c9afb55bf6c68009ce4f956fa936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648891.34/warc/0ae71e1e7129498c613bc9926aa946353c555947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648891.34/warc/5f9495b5412089d126ffa312fdf6522050ee54fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649048.35/warc/28c6217a137a4218cdcb6b242f44ca8760d7a750.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649048.35/warc/7def52daf9414f1784fe937f399a2265dc086e41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649133.8/warc/52d8b7c2aa9a0c06571fd7033b18f6ccd05ec360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649133.8/warc/e286d1ac3a322cdfacf41e5d46bad11c0c473205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649807.55/warc/4e608eb84964902f40f47b2b66164960eadec69e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649807.55/warc/ae2435a1478e6c448cca1c0571b81a4ec1df1f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649807.55/warc/d80d9185c0a34cfa493d2947cfa414dbf8140e8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650193.38/warc/0b2671dc92470fb6e21b475ac6148e5aa8a548a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650193.38/warc/667c5e408aaccc78837e42e5353866daed4ec10e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650516.39/warc/744c925c0763658e9f449291f76a2189e6e72928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650516.39/warc/85525fee996b09db066c585696a0ef206125a5f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650692.11/warc/20cc462f819831335153e25c3e77a71c2480183e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650692.11/warc/85efd81c19a801f8c66c2335190bf2995e52f7ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119651018.25/warc/2d2364b1e7f020ca461c310e2646e50cc45f9f02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119651018.25/warc/8490d3f743f2b60d1f9cfec33a21c9aee9688059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119651018.25/warc/e08140ea04cb101e463d62cd062677752bab5e5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119651455.41/warc/006a8ed10ebb370de750abe6a7b70b243e7354a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119651455.41/warc/719d0de569a0605432c2f36910d55cacc4127640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119652530.43/warc/320128d39f061d1c793b765f662f3350ba76e19f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119652530.43/warc/768faf077003d642d683b4af1d756d0c51179f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119653628.45/warc/7b6631a85b060ca158beda739e4df281af6b6dc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119653628.45/warc/84f40a5d0fd88fb6db9a866314df41d5e0fbf3ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119653672.23/warc/75e3593b4b3135ef24a8b1348378718550fd1f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119653672.23/warc/7f4b8acd5de224c629a11ed166857094eff403c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119653672.23/warc/a9cfb3fafdd85ac73671fe37e7c403b8bc1b8b37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119654194.47/warc/0c01423bf519742d1c785366d37c287b4ca646d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119654194.47/warc/2268340395fb12060fa50f6039fd315ceba5db73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119654194.47/warc/5b6267c470cb2d391c7cf3172fe0cbe1090476ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119654793.48/warc/32dde122ff4e50be7c4c4b625b7c358bb287ec68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119654793.48/warc/9e0ea063961e2eb52ccf00eaedad3f6218bb3fd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119655159.46/warc/56e863595be5c1ccadd6e66c726f047e75b3a47e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119655159.46/warc/64ed7d7a152c2d177d71f826cc3f7cb0fb80a8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119655893.49/warc/50cfb21b8fcb7ccbfa6b5c4ab9cf2cc5f2577388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119655893.49/warc/a5ebd32cd0cef967eaea24be2f3b607fb6e49793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119658026.51/warc/6fa552ea62c6fcfa2f8264225f6d20aee1637f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119658026.51/warc/8221dac36db27fc410650aab625f25bb18f2aaef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119658026.51/warc/855000862238146cba0ef457cde322242b3e2df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119658961.52/warc/6e190f9c29de5dda20d25d957e2df4ad7d31df6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119658961.52/warc/c0c6a97e5a987ab580f4d9e4a947628e697ae808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119660038.53/warc/547c2e6af44389dfe757bc3a3d72f6c52ee15f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119660038.53/warc/c1fddc671951135998731ec2a8f50a2a24c2f641.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119661285.56/warc/43154da8152a44ee65e3cb4f600311285fb81efd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119661285.56/warc/605251fd464ff26ee9afe48efd48e65a86e634b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119662145.58/warc/581dd51d9d7f7a3702a81e41a4e5c3b052fba91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119662145.58/warc/58324f24ff5a28d41193628a01cd356742f901dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119662145.58/warc/eb626f929486b34a93a1ef99bdc0b10d6cfa1c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637897717.20/warc/d6ea2b674666e4e9a8d79672b5a639c09bc83c07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898119.0/warc/01e72f305a63f7b03d553eb9725d1c7e7cfd065f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898124.16/warc/2430030020a92d5a53843640f6f43d8576535e12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898124.16/warc/59a875df1ed4b7bd617547512791443f52038c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898141.14/warc/54b67a8b53994ba6eb77f40436463a89bb3a2660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898226.7/warc/1bf7d9fc86c258eaabc52fe30ab01d408bb61141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898226.7/warc/ab8b98d8d0a4ccf6b8d7e226178db9166731e216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898477.17/warc/43df6245760789dab7290b7688a6bfc707440f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898611.54/warc/120fa190ada6e650a99b53740e0d939e89492cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898611.54/warc/54abb368be1a9630495bb033f4046a598e14b9ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898629.32/warc/9ad37a9ed2aefd98bdcdc3e392c2bed7f3da29f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898644.9/warc/4bb49365d7aad38e8a95a2e9b759cd8ef7acfff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898644.9/warc/b77bafb2ad30ca6d39e8841164061754f101994f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898751.26/warc/c55bcd21b25902a3878f9697abd1b41d33f8a229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898779.5/warc/9816e763fb76efadb1234f60f64bdb0243047be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898779.5/warc/fce513d93178c1cf941b06b061a649dc2694cd6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898842.15/warc/9a00a1a85e1f8c5b46ac67cca31c33551f636c9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898844.3/warc/1e9de95996804868d44dca1bbcb177dab80ee589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898844.3/warc/5043e77c0af56469c7988637aa09bc1d623836f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898894.2/warc/0e276b3ccaf5a97c78fc675c83301568113380ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898978.59/warc/33e5a6ccf30677336814c73c6213212e931b0616.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898979.18/warc/9d86fdbde481c817d0a3658505f056b7fa783799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898979.18/warc/a781fe1273d7f4876b2d3a0e387624c342cfd411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899041.10/warc/8bee17b9f459b10be941e551bbf29ac9d3b1c9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899124.21/warc/053d5c4f126d0978e5f7a20078ae9e1980757665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899124.21/warc/b44da2987d965c31c09a191f6ff42127f527a180.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899132.1/warc/14dad6e6ca638515d7d217c128b733d140cdda8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899290.52/warc/1114b483f3dd5bfc758953d06f42fd2075e54994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899290.52/warc/95618bb5a620c24adb841d96221eead9ed3c9f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899531.38/warc/2685fa21a40dd24594bdcd7ac402562188101b74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899548.41/warc/12f4a9cdbffda9316f2dd6d649ce240de9550043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899548.41/warc/4720fbab6ec3d114e10fd972359ce3afb8db3b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899632.42/warc/99aa4c96f94f4f8a067a0071d230bd8ef82294c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899633.25/warc/22ba06f008891fc77a37fb16c2fcf26b00a776ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899633.25/warc/842391ba74ceefc8a87923f1c383997e2fb96643.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899701.11/warc/b4e7302d23ae1277ef28ba6703597d5d5baad82c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899702.24/warc/7b844c0e07d5b786d1cab6f354eceaf33c238a82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899702.24/warc/b926de28f2add50c6de352d951da8938f17838a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900019.55/warc/66ae0a7736249e718215f07de9e233dbd6c9c780.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900024.23/warc/f525e20669c33e5a3bcb8c1806c54564495bd357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900029.49/warc/5dfd6ee8c91b2bf4e9d19d60a7409f11954678dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900029.49/warc/f6e95c3f906087d1b0505e601b9036b7afbf3b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900030.8/warc/3acc416fc3b316f12873a40aca6d8ebab5896145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900031.50/warc/2033ce804558ae8b6e2759c1561ab18ba845870f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900031.50/warc/2666249e35635f864d51bc6ed6f3e51cac6a8ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900032.4/warc/fbd7986f2850cffe563f1560f5d425fecaca9399.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900080.19/warc/689ec910b95688e0906c9658829017bba7ae7f35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900080.19/warc/ade49aa900c80d5372cee2f03e74145e2f19e768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900160.30/warc/8159d58642a8c65d65d81e69c612f94bccdb95f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900185.53/warc/17532dc7de801ed85c991cedf5467fcda1332d97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900185.53/warc/46e811ee7ce8668c86fe15e12057fea654d25a1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900248.22/warc/59bf8cb4e5f54803f2263f413e857f9fc8ec5cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900379.37/warc/2cd0b166944d0ef6579876e80965b2f2fda9bcfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900379.37/warc/f16b04134122d39c352a58fd838955de54c7b137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900397.29/warc/82be68fc1fdfe67a9a0454439f8066076de63a89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900544.33/warc/a7821221ac209fd0194c08789f67d65f87a619d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900544.33/warc/b382273fb793facaac07f2539ce7f3b6d2a9c239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900551.27/warc/9f096b56337c85873808cf41581d579dda9320f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900857.12/warc/e122ec617012ca00b61320b4638e53fa2d6c5776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637901687.31/warc/08ee6d7b995d02e33be1190e7fa2ba0cc5e32962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637901687.31/warc/aef7eb2baddd0bfe9b819c4db3acdb7f45498eb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637902725.6/warc/377031a9e02461782d37fd5db9188d6a67d3ad7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903052.34/warc/ce0ac16cc1e5fbd10b53874666a78d34a6f55bad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903052.34/warc/e0a9a495fe002831f6ff4fbe522fb5c65e1beb80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903439.28/warc/46993b20253c81d0909f56546ebedd40367ada8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903638.13/warc/73ca08de519fbcc03cc1f4204cecd67712adb238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903638.13/warc/94662f7a67606cd4ef09378970c4e487ac37ee5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903893.35/warc/572baaf992a0a5fac8663199724fa7e95fa4d409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904367.36/warc/09df3d975605e38833240cbe59623f24ba8e59d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904367.36/warc/9ecb96df8685aa9440e779c5e7c601b6fab86373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904485.40/warc/1c7490d6a04c41763da69073a50636c1e24ab895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904722.39/warc/89ad7c87fc9db0b732d96e494d4b12d3cd7f8202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904722.39/warc/986830fd101649d40f8c4e8f5b7871c91a0c815e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904760.56/warc/32a77f1b7d9b1ab7df3618589fbb6e914140bb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904794.47/warc/1ac6366d34d858867b761eee9986dec784bd90e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904886.43/warc/47b09b062a53a5ea3b9c59a1e5f338cde7f94f3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904886.43/warc/d6ae03292ffa7723009185474cbdd954cb85f1a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905189.48/warc/6bc41504a4d1068dc5b41c3e8e1fb047ccb55699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905590.45/warc/1a14175225644d38b2e700b4477bcd04372c4917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905590.45/warc/a08202ef1925b4af91948784521b44723ad45aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905639.57/warc/1eca08e2195c665aa0d35a5d00d975edf47affff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905803.46/warc/1f9b702ade7ed938308b8f3baf35afea232eaf24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905803.46/warc/455b99a26ee226a603c86299c6cdccca1ea9bffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905860.51/warc/2189168f5144754fafc5e352749fdfd87f7599c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637906433.44/warc/691e85ee68384bb2f595bb4203ffe9298a7a78e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637906433.44/warc/9ee6bb4eb55e770212f54f4283bd88346ebf4482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637906909.58/warc/2ed8c2e1a1710cc89509edd3df4f66d4531ce92d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637906909.58/warc/c8d35715384a057031927b09d0e66fbd48cdd5ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/143c547676c11876e9ba54938a4924c265d72256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1467c4e36e3362feddeab7145e8128502cfc06e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1571d7ae11a0ee190b0b4bfcfd212bf84893e8fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/16c36eabcb0ff53af09f492368929bc36537ad9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/179b0ce92e98e07e147d1894810f40d62fa6ab38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/17db697ab4b93f52dea0432a084d80fe1f9c642d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1945b42b6e2c3ecc4a89cdeed9c20e4802f8c125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1a063d855f43f432f5f2fea45500ed2d072cfbc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1def653e160361a258183a3066559fbfce584f4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1fa0436d32867971e88a93ed96d2e6c029dda87f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/2239575a27e962035438337f2462dd81770645f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/232cd6dafe10ecef61d80d31c519cc42539d1c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/256caba67a28b5e5e5cb9e1f1ac0feea33cc5402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/25749d64f878d02ee6e41eeaae2d7106ce7cde76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/27193df6dc9064b98d7d7a157f5465e5338461d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/2be307f00c851c43eb9642847bc4bedc17397a28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/2bf53006f5ff2b7f431e6c0925999da01cf1f078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/2c8551d0d37f7baf6e963f69fc5e50f14b4d9a34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/2d8a539a70727daeb2fdd56ecf02222ae2dbff5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3003c9b28e01abfee5c9a566fc33635493bf61f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/31294897f4b992a7ca619c1ce16977fc66a9c559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/32b465bf28dbb84df86bb841f15cf5f88b28f9b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/32f0b823bfb3d19ed6f5fbbf5125f9a3917ba03b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3399fb03e8e4d0034dd29bbfbaeaec7919660ab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3430505079b0130afd0d21418a8255d6a2016b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/343d9b0f63bc3f6b1f85a5c0d628071a8818505c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/36584f9cfb00be39e3718b61e70e42e30c8f1c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/398a4a2902712d48787bee8640617e8766e83b8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3b6ebb2906d35359a48b4244f6ca9b980bab7a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3d3ad6815f0ae38ff0696186afde49544441cdc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3d745612d7fa69cda98bd9a1e6311092e51053c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/40036d2a996ab3635a3ebde850af481c4f305835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/405a232e8f0d73a111fd66363a59b725b9ea5998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/40e0839196a8296c64f9dc092c6c33c9a9ad1ff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/41b87ff5fa006ca96b57158b2c9f9de4501b1d25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4277f64a348a45f0efcced485f241ab8b87fd72b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/428156651b349044dc0bd76898a0b780770d99a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/45150e86b4353c32faf26223bfdfdc248c10d357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4964779aaa3430b24042605c34c58eb8d95ad803.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/49f2ba029d75e03cb3985cdc664f40ef29d18633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4bc997271d33cb350c0b9d66c5910a8f708c4d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4dc05cc879bd734769c733a0efa8c2969efe0f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4dceaee7196b3e87f19a95beb09c85dd97e8f021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4dd7583978ff834310c95d5709f05be150a13aba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4ea041628c3bb94535366059f15bed4c8c0d0c57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/50cb8ff8bec612190d596eaf6083e0c6b1ef6d76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/50d49390efe72fb630b7bf8c9d6dbea3923acfac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/522e48638f1ad7d25c4b84adf9c8e50be36be8ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/52b9c25b289102aa3a19640b821cedb9aaf13b68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/53151acfe54fba5ef728728727960990cf1c8c00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/53ab6772e277f418b5a914902174238ca401b10f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/544223e6aec4d7827851f4d2d3918d781c74c1ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/55f027be64de1a81ba871141031a2f77914bbe49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/56d819832bd417649393dd54099e5a0dea0288ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/56ebc6df48db016dcc0298b9080a6aba3d42f69c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/580c75f28f0ab9cb79112545a94e523ebf6e3037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/59ed205198155dc9bae7c33cbc180c346e6a1463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5b8ea00de1ecc2f3fc92c716e80ee6aeb7943836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5ca6022c9a29b3da0c151ffbb095c39616072d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5cf6efc3ca3694183911ad902db0adf0333b3957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5d3510a0768cb7e44ec7f347a41f9fbb55ffed09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5d7d5995a9e2691774de97ebf49b02f99bf59453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5ea9dfec694b589d4b1b01ac5cddce3846a59081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5f521832af0d67893b8fc554d55f1426ca52c96d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/60c0cded91f77ccec5dea35d272cbca7eec9624b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/633836f1448825c4ade465e239db4abcc0e9ad33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/64cafcbabb5bce02960033fce730c99d90523839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/654a4870ef76625e1876e515d03b053fcb35deba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/66f825cace453d186d3bbc65939a67481984bc7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/674aff72ab8abbe3067c806253ad45215bef06e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/689174ac57e1a2e36dfe9d99f0a5c258de5833b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6a882f4d48c8537b7c490c6fc3c0d7d6bfbe819e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6b39909d87228360cf5fbd607dde3b4d33a56467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6ca5dae555cf440690e90834f97431d6de11d3a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6d2c363d32229235d8d4133b1ef8f2c9733b567e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6d3593beba39bda61a7eecc4270a2f6714c08594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6ec817649cd09ba9ea181d37aa459906cc8ceb41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6efba7cf4804a86ae150c1a4798dd17ffc2d69dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/705170a0a7bddc69e58b7904cfa3dd319c2e03fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/712954404265551e1497bf4ca9ff326b38e7d3b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7207772ed49e4cb7c8242790d9a06182007230af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/73bf28aad58f930a8136c99b2cdae46662bdc1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/747389d0b4d1f0aac165542fb4332a96eeb0dd59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/757914c314afd24386d7a32cda845f55439a4033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/761ea274f0ad23e7794907736c61579a0a3799ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/76da536291f6d3c6c000658ffd1b72290c0ba74b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/770ebd93aab19708f6d6b9220ff8bf1026ef254c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/79004b022903021af8ade2e6df3eb7b716f36f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7b6b83479581f9d60636d96b3265fb3682b88f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7d1cdb44795311b02467f2e585548c2c7bda9889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7de9cb7c1a901f707f713d7b10015f88e201b276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7df267eaf17983ebcda299091e7e26eee0fbe6e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7e26d711d6f0220870a0cfde67bcc7ff408d7545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7e5cd12ea4b99cfbe198ea8a5c890448caed94f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7eb0758fbff26372aeb8d3740d2dc2e386ef85ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7f66be7f90a31ea6fad201c21874d9349bdc4214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/804354a52279b30e205f5acaaa3a043d9768d1be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/81aabc844aa78f6344d14d1062be44e591ec2f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/83e9d54b72b878899accbecb35cc1d8bf2417eb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/84371e8919086e83878bdbab85be74245b7eea33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/845ee40a0cc9a03b9f6f5fce3a09abd032562ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/84a18aa9a5e7911c0412eec03add4b8550b1743c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/894784240c944cca29b6428398fb12eaf34fdd32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/8c4e9eda0af12e5e665e13c78fddf342d328738f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/8daa9d50b35f123c944529bb8ec993b5e115856a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/8dc1eec4eb8a4b235ee5878ee25fd7573a54d758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/8f6dea56332ca33a2783a1b64acb21728b090e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/904aff1fa02e73c39db584275ae537b68b5105dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/91e45152dcd9b8e6f21c6ac9b5d88b6b4f38e1f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9263ab71a12ddf4a77ba3679f183b2a2ad23e501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/937c7e7e8970e42b8be2bb1ebe15a25b07a0ec48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9418ddc69a89a0f51abee138888e7b0f46359c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/96832ce9e66131c0b88d35e1f0bbe3ebf75d68b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/99c703f01f39ea72d3a1c73010e731f3b06ea725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/99e6b3a85154fc1350efac0fcc79f3f1b490f43b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9a08d191ea53e8d682b7be8f60544153f07af0df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9a9fde2ce9aaa0710a12a70d4108be0d74bd5ae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9b38d4a2354e82e9dc7688a29f4172b2e94f7061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9cd928afea07b4ac6b080a74fb670f74da94c667.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9e1acf327c3088fb4c5a591ee9b13f4eb76bf074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9ef678d1621a9a15b95dd7b825bb847974f05966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9f3246846744dadaf01b79c92a087a5a6f971d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9fb035252c5760c5e412638b607ef5a806c5c53a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/a0d6dbc11dad202138ca2677214f91c2b1a5e4c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/a33a4afb2414315a3e841252ab0324e8e00f9c13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/a3a167836a70ce281d1e9902b9109042f241c34c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/a422fa640f8b02e8ba7a718e44884088cc1874b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/a987ca0a80d7e10ec2660c93fa8bc205ccda5f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/aa0fb49bb36b8a97cf49460d1b21f85ffad077c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/abd4916aed7c5fd29bc937362a23ad2078ac1b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/abe2a87b5b78670da291dff7be6ed2885c829a0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/ad6b995f37d9f1ce4c7a8ab61dddf0112431c41e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/b07aafbd88e6df3a3d5fffb1a622c99bebe4e7b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/b176081630023bfc7a068483d32483c368808c53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/b2ca29d2a2c3de29486c1ecfa711814e6330f57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/b8c76d6acad437f0dfd1a1d933281445efa7a7fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c02f2ddf23662a71652436b8ff15bfaadd6b9717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c0b1335e2767e6689d02e21f92558b9fddb6f7c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c0c0de969097245bd1d1a65a00f3f34448ace69e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c0e0fb78030ab3db52ff5505fbcdffa273be4b30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c14ce17c6ad2b44f72db58211d611b2d3559ab62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c4c2fb2fbfe13ff268b47f6110a850fa5f362bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c5089af8988f5b76b2cc2cfdcc25ef991177aa5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c5d980be0a2b81fe2e028adde07a728fe2fa57c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c98d0096ee016880e63897fa4ee8d85fa77d151a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c99553475d5304ce0925d36e82c6886533333c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d08c08df543ea3775265136b91503869f10c4c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d4e36facd0a5f68314a552f981742e9c527d49ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d5ce95cf720ee24fc3d9a7f20e721ff47879ab10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d5ee94629fd619092b3f083f78186702acb865a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d666f41523d8c11a42f45a82c87867ac31008ff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d782d7580cd0b780694dc8f2a294fcce9a0642a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/db124a28d6105e8a04a84f459deb1e7e6bce8531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/dd6bdd61be08f33ae13c58e0a3cf93a6e6bc76a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/debcc13b4a43f0b9c6097134efb906b398473f55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/df5aeedb00d68ecfe84536b5e3563db8e6411c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/df5d1e4f176ca48c31e641324cc7bbbef6a9a06d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e13b67fe4673aea77a173e73b143aee67143c0d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e1b291d6d17070f9d86eb055a149a15517e1801b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e29cbe9c1741082cef94500ca8f1df16f1234a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e2c3785c398f2eb6c9624517d272f5b2257132e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e3306d45907c40304e05ae6cf0740701aa101f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e594375b062ad5edbbf32b3de0fed4275e825d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e6e31a3f25b058f4180689b24c174aac56da4a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e808da4a4a1049a25c34f4cd9564972a3278f436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e88b9d551b485bdbe9ab07a80c7d47735a024d0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/eb1ecbeea2ab846fe569400ee523a4083f49d2b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f0d1c6c460bebed343b8d94f1fb5c5d1673a1e08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f108f6719fbf072863c2a7c6652c46510544c6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f28ae83122baca9eef7d99afe59246144c3fb6e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f4073d7eb5a12b03105c03efdfc1b1c42e4152aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f5a16e6672122b3797acbc9b522c03b190eff4a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f6a6e31709b442217f8c4e50a3e64e70d04f4c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/fab6eac8d3b669b217108ee44ee83ab53c99e964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/fde35b3b3c6f91ca4e05f848ac9844a7a8423bc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/ff6c1b5d90ba7573d06cd778cfe851ff2d56211c.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2014-42/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2014-42/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2014-42/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2014-49.yaml b/configs/cccc/dedupe-url/CC-MAIN-2014-49.yaml new file mode 100644 index 00000000..d82419d1 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2014-49.yaml @@ -0,0 +1,341 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/00d1041c111683b03f474483a57b1e0ecac6ddac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/04bbf7e8e2b64f38d83631da22ebd77d034f597e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/053230a0592752e5f22f459cceab46701bf4b593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/05bfeca10e4e064d5df81cb6ce2c810e3e350197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/06f3717d048bff556452982504f1b8b722264087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/06fd7bbee474bc934b18b39dce44b90bccba9c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/09458f9bca385dcaee996717f2fd83ba06ce2cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/0aa7e1f44a4b69a2a129806f0a4b657ef3dc44f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/0f426f391f9a56cc853f0d697e3cd0dc38358c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/10a6998744dbbf83bd3514ed0ab45960a2fa21d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/12de672f28282104d28dcb5db2fed0bbfc9f73bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400372202.67/warc/c305e2b4c84006f81c986289ca3316cc57dacb8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400372542.20/warc/3211bcfb7d2076fdd5653306431fe706c10a6af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400372542.20/warc/3aac05f21512c5eb2a6fe39e7b85e8610fc2a95a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400372999.9/warc/14f1b1ad40900d3c3bf1fe701d93945fdc287a0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400372999.9/warc/cc624f139d57679ac6f8712b2b6df0f3b606e5fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400373050.63/warc/cfc7980ee40cd59b6149d540a26d679aa12399f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400373301.1/warc/e18ec0f7906ded18eae9a577aa100531785ba97f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400373486.14/warc/8c225140e89e84a1c20153669bbff80d64d60ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400373486.14/warc/a2f658442645a2a598e2171cb82b833fbf9a89d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400374040.48/warc/921cbfcc1167d494ae1221596663106d905add9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400375630.34/warc/ca51292fbcb3c7d7b6bba6b6d5c095c104ed0cae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400376197.4/warc/1f5b25c9c18cbce6f42828cfa0d4ed0e39c0fae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400376197.4/warc/c3b4dac7e20e29062f6944f3bb8c6fb64e868dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400376728.38/warc/5527d869db8ca5a8da4c0320fb7fc495a6763db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400377225.6/warc/fe116f6c85f1da8443f0e99cd8e33fd46ed1d0ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378232.65/warc/37950731b60dee293e033dd240fbed0a40d30d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378429.52/warc/c20d776c9d8b8e44c8752657b3838871a038d313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378446.58/warc/490f0ad331343ea41cb9c8561c6265e260b54f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378724.10/warc/64d104eab5ce5aa2b988739e51445484a5b12591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378815.18/warc/3f81288a2267ea8d234a4e6fa21970c3d024d73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378815.18/warc/5e1794842f5172187bcac91477481a4dda25fc11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378862.11/warc/7653e6740e73c88b1c070a4746bec6cf2e6bd5c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378862.11/warc/94f4070eadfad20a1fcfc7876027a73334ca47cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378956.26/warc/f44ed5fb64be2e61f78a1522710bae139feea9e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379063.15/warc/12eea097e4430d456878962458c5e889e0a5adc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379063.15/warc/ab1aa647c1e2e69daf4b4d163a957e124e3b9ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379083.43/warc/a9bb309b5446d74bbebe26d2a22e3c797b63c1da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379190.66/warc/3f316c700b213a4dd42fcdd09b86c651f1d724e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379190.66/warc/abc65c36834ea0fac13486f3c1df196564afce69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379320.39/warc/846e3536929aaae5c77a5739c615f7e7784c6b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379320.39/warc/9497925bdb14f3242106e39c3c43e83bc643bcae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379355.46/warc/6f858ce2f968103c0e71a46df49a72533d0d0343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379404.25/warc/1afe2d1c00114f5d83fc76c3ffebbe8ab59fe857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379404.25/warc/5903f74cb1b702ca5689f0973beebdec6a207da8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379414.61/warc/3dcdfe6d47dbdeda9bd5d3d8ca56aaf286fa9d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379462.60/warc/b921c9d70ff564184a38a6348ee70148c93cadaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379462.60/warc/bd18fed2c617f5dc53185af6ff2f19f520479304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379466.35/warc/10f388b9ac13b856b9305c528f3e763be71db599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379466.35/warc/7f63e6cdb322c856dc510263b4e877adea47bbce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379512.32/warc/d8a11ca66ccc5d0c602005c638fe2777b2b2e619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379520.45/warc/2572163608d1570bc99ef6cb027baf072969999b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379520.45/warc/34483f1b55e5ad54d7b91feec738dd68373c6961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379546.70/warc/509c5f0650be372e161076f65fa383fbe2391573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379555.49/warc/1aa27de8111498e9de66b453983ebc789be42372.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379555.49/warc/52adffcd91c051f84912112d7e32aa5fc6a57597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379636.59/warc/008f2a2c5c93d0506f049037816d09dd78e05585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379636.59/warc/f4a8cfb78a3dc924d95a9111cec4c1fb0af693f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379916.51/warc/a52d8c07a8afc902b180732e380c257dab980f53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380037.17/warc/3057c716754f9236e756f3e661075f55209f9259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380037.17/warc/6972e7873efda19c37263db262e56072b9cb6b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380068.47/warc/d454cbe24b0a9854a00b32e8d566b24e9cf17c3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380233.64/warc/48c434299382cb28d46ccaf95a87d1e614f02cfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380233.64/warc/58608fc53acc270c57548fd4c8aaa70cd3f0741a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380236.36/warc/4e93e7c98b30660a011f3bac9d3235b8b6243fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380236.36/warc/90bb3d34f5f75df6afeea424a37a2d00b46e7f02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380355.69/warc/32c28f144754332b4ec6388ff2dc8bbc9bb4fda9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380358.68/warc/c4ed2236df06342140771c91bacc0607e4e80517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380358.68/warc/dd474d78cf45717b80f35c9ffafda9c8fa263cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380368.73/warc/fcfcd5f5c0014f45c2c655a7f63ace4207bbd062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380394.54/warc/278f8bfb5d38f375ad0783322b571be6d27afd4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380394.54/warc/bf7e1b1d2b2d0582a8c7c81766958ce0770c00ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380409.19/warc/460d9363be407429788aafb15b36aa4bd3c1580f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380409.19/warc/4c3ee407380014cef2a143d5943551589ba05941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380464.40/warc/990ba543c38579facfe374428f38205a6811b233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380574.41/warc/8d1e017a50b322cce5ff2b292ea6f3dba413aa5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380574.41/warc/a7c04a253a64c8f5a7e1c619a76dfb0e2cb34174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380627.44/warc/8841729417334fd825098c54cf76777c0993c791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380638.30/warc/4d27ed66868f24870d9fac28fbd1140d9b01de96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380638.30/warc/f420dc5ea94561f101653de5170f1bc045fb0585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380683.31/warc/952af9fa8592f9d87552bc6297b0bb5dd1195fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380683.31/warc/f50860143e0961846ccfd3b5b7b449940dbb3bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380866.29/warc/ed7ad586b6b2159d6b404d3d81dac99c1b0ebb63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380948.74/warc/1b12f164c231472a1544ae6b4273b7ab2153ec3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380948.74/warc/ab6668de5677d06d6427abc9d09c4b4a70cb983c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400381177.56/warc/7c846ee5c267c93b3c8260dfa8f5c33c07db7592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400382386.21/warc/152567a5550eae7a02dcfa9e5cc8bca192a47e0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400382386.21/warc/3c6e1cf9f63cd6a3c33cf3f84eec90b31f216ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416405292026.28/warc/bf68a7ab2fe5c2dbceb1f3902bf72dbde3b81290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416405292026.28/warc/f8cd1ee169930d20d49919a845c77a97e6005832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416405325961.18/warc/a0b9ab3f2c36e47df94c272707b14e3c4fd73f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416405337312.65/warc/a84beb1a23c3f41dc41cc0acf25b202c7217283e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416405337312.65/warc/e255fcc621a3888ae777ebe99c952216622d5ef5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931003959.7/warc/c6b216ec64cdaa6abcbabda2c10f5bf09adcc7ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004237.15/warc/05867b5a8af94cea9578b0ea88bd4e332447812b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004237.15/warc/fef6ef10e80e255aa63ea0761d4c9c821f90f3b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004246.54/warc/39e1df85ee6e616a6fdd5b107c0b9ccd168dd088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004885.2/warc/381dfad60640015a1f0abfaa95fde58abd9db1a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004885.2/warc/fa9de6bb81ccfb040b022d1f61bafdc702d74dba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004988.25/warc/82860a0d9c60044bdeef1b05bcea27a4b4c4f6d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931005028.11/warc/a0c0e1337d7c4ed59097096f087bdb9f31a6db4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931005028.11/warc/b402ad64a4568a63ea5545db5f7e54a4b76ac865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931005387.19/warc/c1eabd4339d77a637079a30d5df1fab882fdabc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931005799.21/warc/46b525d14139a67841f16ec4658c361f9cb7dbad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931005799.21/warc/7f522d19fc665f8f478979ba4e448c6cbdc7117e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006064.45/warc/9ad6bb5b8a806d80a84cac9b8ed4c0b8164a776f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006593.41/warc/d0230c314e509f62d3a226858db26095c873345e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006593.41/warc/e8e6257d76e46bfee3af1f06a03d2704f727dfe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006637.79/warc/6305f8d59f7b874fe4b94b2bae11cfc993ca659b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006637.79/warc/7f103bea00a21a517094c0b5807fcd40c7c148c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006716.69/warc/aa96d1cf25b0fdee6aa44800417e35b236e5c9ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006716.69/warc/af07077fad7eaa6aed2dbc156bf800b9f844d7be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006747.51/warc/37f8d3ec5c2113074a87623be29012096d1cb11e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006747.51/warc/b5c85adae7232134552f9246f545a649f9abe6cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006855.76/warc/dd2db65cae4468527f3f8f8ea70e66fbc4025644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006885.62/warc/174a61a416a6b3cc031e55375d71c814cc12440f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006885.62/warc/5aec2483e24a468f0c5eed3175d9440e0767ea39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007056.0/warc/3c62be5d8c903f39f9d4b47174f5577a70ff1105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007150.95/warc/58444b271d55261915a7b222dbcc6a24ad981fcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007150.95/warc/6ee9dc5bda6a54641d418e2ad32695f2ae7e8a96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007301.29/warc/720d75419d40161a39510cf7899dec7352f00791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007324.75/warc/06ec412a441127b6f6ddc65c8d65c574dbaad016.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007324.75/warc/18d0a07827cac690346f27b2f356c70180ad2b14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007501.20/warc/45edda8c0911b6e61cedd84ea3b1443ad25421c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007510.17/warc/017b9809620b95e4d93b64105aee84e193435f3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007510.17/warc/7873844357805a8fc1da514bd16869535e35aa9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007607.9/warc/baf560a49773b9152b61a864aee3beac067b3786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007625.83/warc/0615314a4fa1f7f0a6c30c9aad05f27da0aa4298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007625.83/warc/76f24202aa1697af63e6e65ede4a1e82780df07f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007715.70/warc/a23957e6b154d270f465de8d227af4b47241472b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007720.74/warc/1882e8590f49be373f21f8d92404e1a5ee11fba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007720.74/warc/dee218c401442ba397e9d416e4fbb8fdecfe8e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007797.72/warc/eafbd03e1434815906ca10e74adf8b131c0432c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007832.1/warc/674248a96a402b7f54445c72fa930b9a38917864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007832.1/warc/fe23e12f07cfa870be29a90dc913155c6c9c25c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008105.47/warc/35c69104063da7d467979b01ab8c841865766041.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008215.58/warc/4b89bf50de0869363805a948372e6a5e060474b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008215.58/warc/555ea0a3c1488e77521aecd6b240ae1aa5ebf514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008218.28/warc/545e2cae5982174878d4e9323e41c6e761d20829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008227.98/warc/363e0b519cfce1c2b0bbfdd3d87edb9a3c6e67b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008227.98/warc/aef9e08e31fb86e78697ffe54acaa85ab4852e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008289.40/warc/88ea4fece8ba8e54772a884afa5806bb76b24db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008520.8/warc/11ee39a59ecde4f3244112217db4d7414d3132de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008520.8/warc/ff68aa22804bc51897ca56d4a8843c6b33a0adbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008720.43/warc/e67a9e8b91b4600df2aee035a835049626f2a732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008899.16/warc/191e5f58e464fdbb7adcddfdf3bd9df640ce7f37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008899.16/warc/3974ff5deaca7fc5bbc340b8601a00a7da850119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008919.73/warc/3dc80b23116a3bebc719f09df0950b900f479e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008919.73/warc/979e3d0ab19c8ee233a388384d362280df0ded57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009004.88/warc/5f916f290a1625a85edf38b322b68626face0237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009004.88/warc/749bdab7b42baa7f78bc819112b7eb8628df21bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009084.22/warc/c5d24e26a17db55b345337cc720e44a933fc4184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009084.22/warc/e256399d7b1b3d91c63d769074ad86e91ab13eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009179.34/warc/03ca78e82985ddf567c4664164a0dcb74ef13eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009271.61/warc/b6e22ddab2b79e2675e9225676de8ec0fee7ba38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009271.61/warc/cec808dba294eca6b18c48d8266bdb30b1144ccd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009292.37/warc/c21b1fe279291c6becf4ec508fdab6194de095fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009295.92/warc/8069b9ae2060a9fa10cd23c2fc0eb116e9d23bc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009295.92/warc/c11373867e7f906dc9c1cbfadc26c7b513bd37c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009515.14/warc/e96b6bbf3979d5a793eb81ad1bf03f80213f0154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009551.23/warc/c527d5da2b125763d2d3624e14f2e9e9741d3239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009551.23/warc/d7440c780a09961424ba3cde4492d5bbcd0d8404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009751.93/warc/889c730a270687105710a460b8053ef14fb3798e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009777.87/warc/72774124114306823d4dc06ecaef06915fcc97b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009777.87/warc/e80de80b982c7505fd48b89d23931c6db77b1bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009825.77/warc/88b372e2b13d07e1e0d8c9edadcaebfdac95b393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009900.4/warc/521e5ad728552838095eaaf7cc2c9b03e1a7daa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009900.4/warc/7a2240594876ff143588bc696ee7d15d678c094a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009968.66/warc/f9c1a76c7c0fc01ba656a8b32837112144ee6b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010149.53/warc/14deb4dff11ae5c9649eb91e1540bac7597b77ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010149.53/warc/2f0cf5d2ddf8738fc1907a2fdba03b495c0cb4ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010166.36/warc/2917a51087930bb52ee7f3c9b5d963e79bf65486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010402.68/warc/4ee4bdd0fcb12fd16f2330cade17473d92279586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010402.68/warc/c3a81a1fcc10c7ad659a9636f9aef232b41103d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010469.50/warc/9cb52384bbd4346caaaed7678b33762af99eb812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010590.31/warc/0052e6d2e80d85c65e71e360921c019e81e62f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010590.31/warc/fa7f82994eb581c59c4457aa7667a90b1e17c426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010631.60/warc/2f1ed021611db8971395c01ee701148971bfa5c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010713.38/warc/323ede49104d2c641c85cbac22417b0ae32278af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010713.38/warc/c5ce598d01d94556a823ad837c15e7460681568e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010792.55/warc/00fa0f8571c64d6027485635e38aa739feb819f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010938.65/warc/57dbebce6d4921f0e0c9f9f3d1fbc93a4976d181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010938.65/warc/dd02d371205eeb76e66d5e8ec53894951b0c7e5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011030.89/warc/69975b9adfcfd536cc7e2087779681c923eb5f64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011032.12/warc/69872aaa7cf3324d8ff529faa69055cd3616a5d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011032.12/warc/7f5b7768b99c95a72ad176e57906b2a507e300a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011060.35/warc/7f2c1f85165bd93401013319e00bb13683f3a666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011060.35/warc/e54625b4099f740d3cdd4c1d7a975cdd1e1cd24d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011456.52/warc/52f9f34b0ec922cd41ec5f47bc23553b17aa6212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011456.52/warc/5619475e283c24edecab42e030f813485d971458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011477.80/warc/62fa8a8ae10c533b63bf44f61c8a2ead14f52e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011477.80/warc/65bab9fc26e29469686f20b03f28567018a06226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931012025.85/warc/f4d91aa513de66ab846e06eb734082986e6f7989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931012703.33/warc/0db9cd5e0b0076f2302d25576d0e76a38a288fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931012703.33/warc/94db28bedd7db43c507790de181e075f90097f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931013466.18/warc/b5747209fbd24967a34ffe7e90a32389a2d9f39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014049.81/warc/ca50aa4a7385f171d29916e5a64f12b040cd4ca6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014049.81/warc/fcffe255c0c540faf87dd84ba0897db9bdf15a71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014329.94/warc/fca3bfe8994d7b6cf6b15fee7aa41e0beacd6119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014369.10/warc/5aaa175fa8fb2bf174a89e49fa3f1591fe6cd4d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014369.10/warc/a127fd688c4cadcb5e78e4e36818669bb4e08f13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014369.10/warc/b0f9dee89d8c5209793f0493efe9b82a5e22b5d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/14892c81e04bcd0dbe030e7a1f956f3a30c19e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/15718430c694b8e05f4215edde7fba8102da4781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1601135a9207a713bf749a51f18659b7b56cdeee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/179239711b3c10b97159df920d1aa671927804e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/179f18ba46e4183a54825071f7e3b082bd7652ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/190cc5863ee51cb9507b4fcf25312522c86ced92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1d84f17adb725775e770ca7729128ee1522144cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1e98f26c2d46ed0fea055f515598aa79354ba193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1eba22f7f0f8a071e35458293d9b4340abae4d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1fdd16dafe471f8a3cb38c49b7df8b14e9ca85bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/2423800935ebe2f15831f2d7c3ab2967646a3bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/247201208ab703dd8c263f3c6791f63971de25f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/290fbe5a3163192a88ef121701f812f5c7695fd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/2a29bd4f3b5dcd0323c91f471edb7e5be75b751e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/2a438e4554e172d12e5ec00c41d5436660c9b97c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/2adaa713859ee810152c1f40663395e6aabd151d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/2ae2352b4191ebfe58dfc5c1331944f0463425fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/32a6d156278ff22e18c0defc90bc0a0faa250a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/33398c29e2c2c3ea7c6eb049a49bb32a1469530d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/337667eefec9bc05e64fb331388deca2389f3d98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/371970b515e0149b47f66fa5f562ae79ebdda5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/395ccbb6fe796a7b052860340e04bb5ad06e243a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/3a2f5ac46c1591ae51f57e6250eda03269d24b66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/3d44b8777d21b7fe653ba66dd248e774a2e7f190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/404a479e5c53e9263be133ef8bb2675ca0ec0de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/40d5b798166d0270ec1cac6df4bfa6780e31e3ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/43175af8fb1822e292c1299c671154656381aca7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/4445943bb01fe1b768b231f0d28a865795df90cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/44f41cff78f5fa45b3ee5738a59ace44dc1f7102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/4603c70588ec82dadb8692595c5adf4f5e449d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/488e13cf587f0dcc28dad88a66bcab6ae72b1d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/4fc3e3d41b6eaeb2a25afeac021a78ff5076c90d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/54287120e0e2830d9c185f5654f7aabaf7cae7fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/547a1faab531bbef9b5ddc208086613e34a2dda3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/55aa3124fcef410b2f6dc29c416f8ebb26eac221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/56deb5bb768dea7dda83dd42a843fa9a9b6b0692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/5cff44341bbd9b7b6d1e9e92d21053747c221b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6014431fef4a95785c284a2741506b21ebccd4a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/60726f59bcb13b0bb0245e38a1e972430ebc14d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/615c330dc16b539e7b111b33c4b3ae2e21f11c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6468664adba7f7b5f50302ffa054c0c1259e5e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/653e420ff9b1a28c2a338ad2c4f3ac1d78c756cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/656da1c1032555f42f0af29684c5926a49548a7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/65c83e5f1a07d42b34949ee8e5b3299a3ae30f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/68dd337111d790f124acba4dda3b72829bcfaf92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6a26ee41fab23f1a2cf8d0cfec244e1cd2f62f3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6d9340939e5ca743ae91e8d592f94b2df9fca249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6e5e033cc5b07f51e5f91b942b8e71d0b1d9cd41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6fca9c1fa6ee41af30087daae28c8286cd43912a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/71cfc310385ee08e7eb952ee834516b21308d686.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/76065a9f163a7b9c924edfc19f365840c1718eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/760b6f14ff734d9d795d2b023dee4fcee7b53c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/7a166b2452274520c1d371b27ca778db7859bf9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/7a6da8b9170d20093701e6572acaa651b07c159d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/7bcad6d9694c9fefac8f73d8ad440bcdbe81c796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/7bfb9700c20babc371ea0f867b0b5bc9fd3b47d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/82086df3c8fa6ef807a6161b4426ceddefce95e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/837fbb08e846a3ebb3f0a3ef31ab9dcb079d5966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/8fc140b97388afa5122dd2f5fe334f96884cfcde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/90af538ffea1b4919617aa25e08ccae77d466496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/96ed13cbafd82f5da2ffdf407d6ed370caba64ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/988b7b9acefa39e10185a35e1fdcfca0ec6c5987.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/998c81c24fd36e84312d9c28cef75eb4316b81d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/9b6b784e561993c0781eca1ed2f36b91d6f261a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/9e802da78ec4fd8a6e31a19795b8297cc37cc269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/9f8804adbace7d48496c74fd7cf753fea2538eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/a1d61317ced38796997c0dca5d05c9f529fb44a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/a472bf16c6fb6494aa284a9732d354fc0f82a5db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/a61c19e0db61a613c0e925c82fd8b546e177e4f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/a65cc5d9281d8ae0a346b7204447200a70883349.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/acf2a6053dcef2a55a48f312171141eb40a726c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/b3a6282c36bcf93d009195936ec2528155d4cd45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/b5aa1fb29ae6b11ad0ec21d714bcebdbdad9643e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/b912cdd323c410ff8bc4971cc2c832815779954a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/b99b97c497a33d2e2c34a0ff109f917f6c5b68f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/b9d7ffc7611b4850e11d496b15fc29cb40b1c2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/be189230661f05e5af73960ef1b581212507821f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c183780400c5c17dab008b80034a621233fc3629.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c408443ea7db3973a2cacaa1721ccd9e96abe53d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c58e71da4d806688fc2dd1c473041f28fee2d7a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c5de6c8375cd4df9993751b1c9114e788199113e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c74235ef807e5a9f79ad5aeb93f6f02e910b9d21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c95a3e5e0561a9bdbabe4ea51134d8652a4c07bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/ccae7ae9b6f464628d20728e0df796a310aed863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/cccd1b04971eb766098f9599430c4bd75c49d9d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/d04d523a8eaef6a4a1da208ef81ec04e475f3bba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/d142234dd527e945a568a3b029338460e2e7611e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/d7b1019c70d34057b309e42baa3282bb83769736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/d93abf1dceca9abcefccb73a31de187589b099db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dad33a74b254dcf61d2ac93ee592c48f5d78813d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/db136a70f64c24ebda392f0a2f13cb7df320a6f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dbb7de9690f39b67e85f48186435ea28cd916e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dbcde0314ce847c0a7bd9268892edba79a82efa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dc714bf60fd5a030f84ab17dcadeb0a1c4f69a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dc8df94bc1dc94a92fe9cd9faf4ad9fc03ed09fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dd60ff6d85c37d0bf8e33d30eeb4b81d4bf85899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e0365870d4d91c350c4efa664c5d2a08ab198004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e280fbb8914c62e91ca78e085a17a02eb28b4666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e348b0bb3b0b74fe81554f3a7c8a84b0ef98a2c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e3aa34340d6f43c0f70b51efb97044d4afa7fd88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e4995f7d4f388c03bc15cb1a8c0c03656734acd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e4ac4de55052add75cad9883b5851bb1b8dd4a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e7b1bd19fd8cd5de3ea2d0eaee1d3b2ec9dd6990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e8401d1bd64ff44a0c54e7d9da499fc0a246dd68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e93e017f09d21d68edb30856ee9964be51d3d0ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/eb563cebd3c3750dee44404e83473a30c99dd519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/eebeae1fe0880503232342142320804449248dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f453ec44dacbfe2d7b5a92fa842b6501d658862b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f499d9dcf24030a746ddaa0a697cbdebd7fb5dba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f521162773d6185184cc7c1e3dade5f4433cd1a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f68be817cbe7b48b6553274bf8236dad9a006b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f6c20744c950836653564cff52fd3048c2a5ab2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f6cba2f083902258de4105ee225e7f65503afe93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f7fa9753d5da52ccb7e5eb60abcd7ac5bd456b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f91ad474834d0c8c600863560fe462a45dd08ee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/fcae552e41d956e45d167653f67e633764ebdd4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/ffeec43fa9a5a69caf6bb1060c0eb5c659229211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/fff754a56b4a2a6c1e7a6fb85c5530ca0bc94d19.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2014-49/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2014-49/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2014-49/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2014-52.yaml b/configs/cccc/dedupe-url/CC-MAIN-2014-52.yaml new file mode 100644 index 00000000..5a3066eb --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2014-52.yaml @@ -0,0 +1,458 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/005d1efc732719a9bb375c5e8d786e36583ca8df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/00f9a06955248979664af8635bf28aed90d74504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/01a038061fe8847e4e12ff1d347c01811bede443.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/02fb8a7e2155db129dd8ea7b81c505cacaf2eeba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/03535ea6b9c0b16c32f296fd5ade492c2698e5e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/04b7b6aef20a389b9f4aed7a2bd5ac438418b50f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0599b28dd5803715a29c5a68dcfe17e2455328ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/05a4de914219cdf9a2cbe181b0fc2d38f83980e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/063bac7f20fac0ebd9a4f99258e00aef2ebeb1c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/06464af511eba1db87d8fc8eab122e3f85e79a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/069dd0b3b77f9d84afb3fcd732cde24b8bb3bb53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/06ba24100e9b881cf64afbe4934100f0a26b2f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/074d5f4a155e9a3611b354f46df3f6492bcb1c0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/07f0ec5454a15cf643548c0f21e1a6bf8aed9a4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0850859d554a46fcc95c4d3ef07282af348567ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/095a915fb84e410d1cd8e158d496edd86e5b66b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0b52b48f76462370d51f21f36f2007d843c29929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0b7296a08483b3702dd664106189e17837c4d630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0cdf3ce821a9f852975ea24b3230cfe5b6598b7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0cf6621fe96b37d62011dec11b669c667b3e5592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0e56b1581438be404e5c244a7c41907ba3db6672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0e8521b7831d9cf02b201fd32a6f2215637bed4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0f989922baf8174eb0b4fd9998d96bc9cd63defa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/113a38ab8d383ba36b69d479cbdc14a3c32710a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1180442f548bce9b7ea51cb99adc2f753359af73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/11d47d91ec2d93f1b762cae46abb74aaec05ae3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/127dc95d69ab9dbd96187861e97dbd0823c62bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/12ac7f006048b77926c9c57d7028fb625b4a2361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802764752.1/warc/72a36c1c674358c761fdd4469f8a8e1146d8fb0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802764809.9/warc/4a03731cb04727379f986006827b095c2f47bf5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765002.8/warc/847bba5becc34008c78e6eb1dfcf44615adc8eed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765093.40/warc/11706e24e74f8cc6aa5502b30dfba34d6a9768f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765610.7/warc/5a6a5cfe8b8d4dda5137414d406932ff13fedfb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765616.69/warc/041659e6587ed8331e87d37d751adf176cc32402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765678.46/warc/35639c282755c13e6b2d3be580fb5381ebd20abf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765722.114/warc/2acf35dc5c886be26df4ec4fe5fb56cd02df0c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765846.54/warc/ed4972a34ef564ed29a351b09e82fd87582a82d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802766267.61/warc/4fc0957b4aa36ca23e0b02cc512597cb1149572a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802766295.3/warc/a1628fd3101013f78b69ff67cd809a908571c802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767198.25/warc/634e3006a2721dbad80a5878c5850c923eb568cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767247.82/warc/dc51b40c40ba1248710fe82ba54e9ec454603515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767301.77/warc/8046b1b6050e796b667116b72c9215b3f56eefde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767453.104/warc/6997571835ededf98b8d41db101c978bae0d08b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767828.26/warc/f827d1fbc00e4d06e32bcc054c80136d3c750353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767873.65/warc/da442139046756d0b98f9ead8d944353daa60e5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767878.79/warc/d7ae2d68016d5662847136e6159ee6809eaf43aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768034.59/warc/301c477411e59f7a268f5a2a910ac65215cc28b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768044.102/warc/690b1fa9074d93a7908725b1efc2d635dfc45629.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768050.31/warc/8e44c2d2faf29ea6403c0d6da3a5dd215cf931ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768089.153/warc/12f756355709c2a4d7aa60099ddd56eb92f2354a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768167.64/warc/a745d9b6d9ddfbea62d50f5a4628b897ead98565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768169.20/warc/574f974a3596a023a13248d013edbd778b8ee573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768205.146/warc/9bce8fc3ec441ac0b63d62be80359eded24bb266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768208.73/warc/448c0db5830bcbf24195dcc44da8dbc4d530a8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768276.101/warc/c8c279ca2d5700dfb49726a2c5459e2217204719.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768309.18/warc/42ab2d815fc78bd823a1f1f25cb8d0285c6934ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768352.71/warc/ba4ae260f6be95d44c5d2795a4c58a0d19314358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768378.98/warc/44e5bcbf9377864e4bbcedbef9e01499f5593595.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768404.109/warc/12df6144d92c2f3198018ea523886cf8a988b8cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768425.151/warc/497eee245b4078a29b8285eba1ac2b98742b8b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768441.42/warc/1882cbdd50768a52d2a1ce6e94d04db0b2ff443c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768529.27/warc/acf7eb25d15f5e2d73085d27ec49b63de06db13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768561.127/warc/402439684009fce17728d3b34857253ef9f16cd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768636.90/warc/e49be0e7ae9ad6a579ef3ec33f3a8c0fbe2cf17c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768741.35/warc/1ed843bd2471da0561c7361b3b9913a0d81d9009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768741.99/warc/d966e21418ba3c4e3551edb521e6f12179007f44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768831.100/warc/252079b4249d8c6d69d9b805bb7b36b089978b3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768841.115/warc/01900fbffa9e67f748bda31751994289c1f5e4ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768847.78/warc/36a13b14afd6790d65f40f68f5299cf972aa245f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768957.83/warc/60ff6672efcbb9f1b0fd567d51fe7c827a407b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768977.107/warc/fca787a49da066d2a0eaf16c7f61faa15484e6ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768980.24/warc/0f2dec2e7ed06e74c63fa6cbc83ee71a752606f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769121.74/warc/00222988fc437d7071edebbe09f23a524185e94b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769305.158/warc/b7fe2a9a477cc78ba28d96ae42bfac8f702e6b7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769305.33/warc/7026b22d100adfb4fbb285b3a3d8b4644b0ae272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769328.92/warc/d3857374aad78117e2659588ffe31d885dedb7b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769373.55/warc/ae33d65828dbcf3557b886ccae698a8a4d932ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769374.67/warc/807e5cbb5b97ecc8d901627fabab743cea659498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769419.87/warc/6892a619c2d40b31a73173b6c753db7ca9d64714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769550.123/warc/0a2a7015a9204c9dcd7a11697944233d6a9e7388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769581.93/warc/a5ae7662483f6451c30702b1b8ed953d201d8a30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769637.44/warc/6a9c318a2d06785ef56a1ebd5d3b30c63e0fa710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769642.136/warc/db7add047ffb9e66b8579ebee920d1e81faf5f4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769685.0/warc/2d2cf7c35dd6ab13da44f5fd7db3f9c61ad8f237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769844.62/warc/b81addbab51bde55f9bf0df12b5db7251c27f3e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769867.110/warc/4723e3c7fb6a8a0239982b9357f51591ca4dbfa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769888.14/warc/b40bb706f9311f9ac36d23693e97905596afa84d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769894.131/warc/fc905344076802dab3ec393b10f450ebf75800ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769981.58/warc/263407fd6da11de19e6f6aaba99c8168814924d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769990.68/warc/baacf0c90a8146538b22a7bc28def78823d38695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770043.48/warc/7987dfad7d73be1ee828b198d1dd3921f8292656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770060.91/warc/6c1747f162e0b219527b4bf9b54bdf72e2a85f01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770130.120/warc/f539ec45e108ac395c571e018bf5e571263b93e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770324.129/warc/f4679527f65267c4054396662619f67a62bea0a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770371.28/warc/b1e6e75a6e0c9adb02deff801a9fc07bc3cd98cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770400.105/warc/89f0e5ebbea01deee76ec6e8295d6eb3c79960e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770403.126/warc/e63ce4e433a197cc47117d776d9b5511223b920d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770415.113/warc/9a1a6788c2f009204246090bff06a0d220fd455c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770433.122/warc/63dcddc61566d70e8a47a4dd9d9c5264cb05d60a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770554.119/warc/1857634dd23392c9c4fa1d1e1e2ed89699f71bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770557.39/warc/d4eaf52a3473e0acbdae848c3e4499ed5245bdcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770633.72/warc/e03d1f50d0c7c0e23c4b5a219a5fb4bc421e5d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770668.10/warc/8e7aaebec21adef0176f22136c250765ba6dbd8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770686.106/warc/787d84b3c81d9aa3513277ccd80ac5dcfad0cf3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770742.121/warc/ce2bd21ab142e76c55d79e765f34dfbff590b32d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770747.108/warc/99c0ece9b2266bf1a318f39c303ff5121a168b0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770815.118/warc/bd630c4ba82590d4c0493564e237b26fd84ca635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770815.80/warc/c9ab1c10696ba9d08a8ca33911efdf80f140834a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770829.36/warc/5ab070506b39eb5181a81c84da8cdb723b0ec807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770860.97/warc/995e668b9d6b12759e9bf242b658e73ec9c6f94f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771091.111/warc/58d7a4586b8e57e73be786fb45a10bbf963cb143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771133.144/warc/c3493227e3788ece64722800b78c8312383710ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771253.22/warc/77f0d98d99d96a23b4b97c8cff055700f23bc2be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771374.156/warc/fda0a24df4e440be334b4d3fb231ea56113aa7df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771384.149/warc/931161de970f286852a3bc2cee8a644584e13cd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771716.117/warc/7436e279f5f6730b1bd2a64f5b2255b76ab8bc87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771909.45/warc/2511a57bdf137c36fef12926a4daf29d8f6742e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772125.148/warc/86d7a91a89179d760782807f00ee18b44d6c79ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772265.125/warc/7562ab0d8f0e4832a01dea897d56c79a5f88f20b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772281.50/warc/2f6d1c9f20c41c2d02eab6dd305c9042a23b8160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772398.133/warc/c8576be77f95c9f4fd55c30832aa06535edcebaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772743.56/warc/30630cc66f4de6728026514a04b0fe9344f3ca7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772751.143/warc/01ccc2509d3ac2520831a62573f86325644ecb66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772757.23/warc/007d83f4a80a8498aee345ba575fb86b761a44f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772972.2/warc/ec05f8c69aeac43019e71811c696278b36ee35da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802773058.130/warc/96fcbec2f838020e91b9e6e18ed9f03937924b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802773061.155/warc/9baf1440927bc815802b3318cd709d77791fbced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802773066.29/warc/f50c53bb0cf64f6e4ba920fce1b683e6240ff5d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802773201.145/warc/f2b727fa447f56fd75a5959796ff3284a7d7b005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802773864.47/warc/06e14b7e1726d1b3bd42f92348f8789b16d42899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802774464.128/warc/7d4ec38a77f51e081fc8ef88772199645883b8a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802774718.30/warc/3cda2b774ad4ed6de547a08d95cf3ac35a72f759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802774899.57/warc/1866c2a42d03158b9f304553edb5c09cd5350fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802774986.137/warc/bc6b760af9b8e3592a3150f364af7dab6974a6b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775080.16/warc/78302210fcd93890c2a9718d4d16885453993186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775085.124/warc/98b8c8b2fdcc15fd0f87e913be512a23d751978f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775221.51/warc/d35705a703cb1b6e44e66a8102bdc7a095733b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775222.147/warc/db8aaede143a86b865c8f543c78a4ba24cc57457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775338.41/warc/dd4ecaf986218908f1b9e92904b73a6fa56fee17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775348.152/warc/a31c82582210ea07a1638a80a756b1984dc7f920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775392.34/warc/d5ad2b8bd38d76c2913091f0c1af0ae92578d3b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775404.88/warc/7dcf3e21ea35d2fc6352bf10e2576a6d8fde5e43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775517.52/warc/70e3c35614396fc9615938459e3f2495e18d5b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775656.66/warc/a6d4dc38f85294c2a540c3f34d318fac5bcd8e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802776528.19/warc/61b869ade791169c39b805bbee35f487dd658e60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802776556.43/warc/d9032b7bbf5b68944bd822ec6a70f065303796fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802776563.13/warc/759ffdae9675f6f0348479587d05e81e599bec4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802776996.17/warc/dd6b4be1d3a132a4c3b98cfb7970506c8ac3e5c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777002.150/warc/e140d9c57c11613fd0ceac8e427bfee674b215c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777118.12/warc/866ba61d26bdc3a4942b43eab35e5f35552db865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777295.134/warc/f1baf2bb3811567f161e32cb7bf0b386507d5807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777418.140/warc/b3b0c68066b465864e6f1955a70f726dd3e73567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777454.142/warc/c46bb7f201b55956dc53e20fbda943f0bc0f834e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777889.63/warc/096284566538927f360977a4a44870989fa450cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802778013.38/warc/7048c4bfa751dae6e5deb95e346be7cb29480fad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802778085.5/warc/d5f11051a9523a95ef5fe31ed97fc4472c258acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418815948154.77/warc/1229daf205c2d5bd38de3b3c66fe0997bb08a569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1419447563504.69/warc/0a3fb8ab840bbf2698e433fa3fd6001ceb3daa72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/146f6b073afc6328aab3c7fede1c9a571c7b6936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/14dc61110fea609ca094c60047745c512a449173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/151e37321b571dbd585c95ae542f0a30242f6d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1652efb629047b0236d0110c654038b293362fe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/17e3658dfb421029a2f8ed9f72560d454691d09a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/182c74b873443b2ffc8fb0eb2fc51a5a86a20805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/18afa6d791b49e1e4af50a4b76c1e4771375193e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/192a9ccc7feefed1a345f3d667fb1b257e8181b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/19d28ed9bf5f92c8d4c48503cda359ba69c43eab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1ad8d5b7741e5f8bdb51de28ea4cfc2baeb2f2d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1b0ac0496b01cde648275acfd669c0a3c5cf2526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1b18e16cbec7f8ff94fbd54e01a55de476e32664.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1b37833e7470a902557c7e6172d95f5ea4eebd0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1d85a082495718f6e03cf6c2ce98ba658c457a1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1db6158b0f0edfff34d806b50c5959a44b79e296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1db68a27498b89597fe308baf2eb7136bedeb652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1dcfd858614de4e7616e310a18745b5ac0bf0096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1dd7754e64bd7b9e43c83391f4a0b08f5238d877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1ff86700fefe994cbee695aec0f93dfb44a12a0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/24e44d15e878dffae3725e092ad276e0645b7908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/24f408a5eec3e32da9fb28f12d203b916d204766.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/25264948042c93b70e60b61cd8d95e6111573bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/25544d0bf86eea0634d39ac6dc45b9ea8cb38770.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/258442d37b319dcdd90441d98fa5cd8bde242bfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/26b6522ddefcd6f3d48bfdda8daec22a53e21071.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/27313af9d61b173785a39673261318448fed7e7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/27d66e2ac1982a6d3d60b2ad3821b6c7dc5cd038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/28edab955054fe262471076b9b153aba22d1073e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2a209095ad7496c663e83134acf10959c6f9352f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2ab4469ec442e1d70ac386c176dd7893bc0d92fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2bbc74e4a0e044ca6742b8b3249265d689c6bef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2df17c38c399a300392ad337500e89567e8e716d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2df3a9a950cdd79dae38d65f736cb98f0662b3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2eb134a0d76e2e3b48632956c14fa89828ed5bf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2ee62088cf967b91d050c810d51fdfe9fd9884db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2f7b2e700590d0bb7bd141f38c67d363e0070f9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2f937f235de7872b0114d99bf7f1942db93fe842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3130843503caeee441b1fcdaadf74da247f10d0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/31f0df61f9e21b4e703907b6c4b29b2931e6d2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/33aeafbd301eb1cb332f41f3e369e9504a0ee79a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/35ac152525b3a32028ce4fd8b1e1166a04bf065b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/36c1c737c8b721bdebb5da04eb2e9262607d19fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/38dd6872cfe8ad87bb2cfdea7c4a06fc03f05739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/38e1409d3896f546c67f2f0a1fc4a05a91bc4e7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3984e75acb00306109fcc603e593cac1aba6bf73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/39daf3731c647279d097fae3e8241a0e186bb05a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3a0204abb39b60d248ea436541bf899b2ee3b3d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3a8f5c3a2bef95453fbce85bf8a04d903ab9566f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3bac0a8b6560f8e0d27d90d069dbf8b5eb830b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3c4dbba38cf7596d7dd7a0e20997cff1b9e9922f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3d49958d39a8458f0acb43d24ff45429573ae96b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3d54524b29495ce54af66f611f5124f58475eab2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3d709e9f2e7f81b230419a272ce77588fead61fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3f6ce0203f181af3148be364e24fe5599a2c6efe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3fd72fe3cb63ea8b9dce2198fd9c6b57c9a9397c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/40ba5a0626474929980e9e5cdcb4cf676fc70bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/413adb5281454cbfef58269518117745dd42423c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4205cb5ebda0206407bc454e6d11e889887d8f20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/423109e760a92371eaf4daa4a93a5eba9b936f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/43966f2b0b2f2f6eaeb5fdfd5f068bf790c00731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/43e858e0e4d82e61acad75f2d487893cb4fe9557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/43f47e390e402d95ae5635f861cbf5b3dc447dc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/442d680d76cd3ab6cc00447e7aa6755c50bcec1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/44489e2f544bd9ccad9229c96d2b28d61ced4bdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4733322dc610e82ac50a826b08d33df62015bc72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/474ba3f886b47864d0d893fb3c297261fbeec7ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4755e23cf11d942eab9e957660cb9a533e5b4ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4882177932296d11579e267ab0d10424f1321ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/48d08c7209f26e445cc040917a4a0a351aa84054.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4b3cd52691377c692b37c06f175643588797ea63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4cee4bcc258b3e4e04e4272b0e27eaa1ea3bbbc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4d574b1326653c4ffdf98502ad66fd09e05e51b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4d6fe4929b4d175510e21650e2dfbc567b9d396a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4db0674119146950d24a6869c8203ffdc09c96fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4f5df2fa857a5e7ab8d33781fefd1ae8a8819a53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4fb44aff1077f99d651b87f126e24851e42b385b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4fd8a456867708b8827451005e1a68c7655e14f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/526c5186ea14848b38d2b21b944e30547fcb5233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/541f848a24d40481eade2ebc605c01b4a66e892c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/546ae8882dd06076fd80fe1e8cdbe6fc9857570a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/554be2de596d663df4a5230ff3a153216b9687a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/55b115be64c92166e424bb06db8c70cce9894460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/55c6bbeff3e54574ee193e3eb72b945b3bede60f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/56bb6b9b44ac562cd7025a959c02c925ec679cb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/574457af0b0b618cd39abbf43637ebee3a20371a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5867f281b48b95102be6b6297e1babc2883e6b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/590a8052fd64b4517e659a00b1849ac51dd93d31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5a71d3a42f87bd7bd6fa68fc949abcbc51e16376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5b5790e9a144d813e9b0922c4bb152f1c1660a6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5d3f0829e8994e395e578d10865da7ee68298feb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5daab5efca89701aea0e86a7cf43b32f7261ecc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5dddabd7564c627f8d8892c74b39ac08b8df2711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5f9ed9bae6e8b59542cf925e0f6c354d1e0dd4fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6031ce711c61091081e46d14c4de50271d554c07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/60eaf1a93e0ffd30d0ba1689f170d36a3a2c5be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/61611ef813d69f862cef0eb6907a1014ab97bc35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/618f6b2bfdb8248a27306312aa4bb327cfac5bd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/638a4be7fbc6aa19f568a2bc7e89d893bf969c4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/63cad76d079a0db13d3db2bac810dcd988eaf463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/65b4db6d798eb44ad81baebbf86b5ff2183febe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/66adaae7b3256079815fa1c033fc86323067bda6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/67bd9fd9912bec0b03991d5f4c747dbfaf10d7e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/680774b5a8ec4b58f4e9f7831b390d6bb4c7baa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/68650428b587cbde3668fa31bd3aa2de4d831520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/692eaacdbcb847be20a88f3d4b25b716639ca03c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6a9d560a6a1f7cea103cd008e8ab9b22767df2aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6ab02d50ceda12beedecde4292efe8caaf7d3aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6bcedae138d5803422497c5cb4bf5e328676a408.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6be94aedb983a785651d3cc33d32fe44d8c4f6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6c1ba2ca49e2477ce1a053bbd3868724f26d0ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6e4fb830cb19e900141d89783ea0557a33f03b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/70b592150a9d10b9982fbf2f94b49f2c34d0d3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/70c19e72e7bb97f345851d4bf4ce16d67dd61b38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7117471cefbdf82d4e722b754d7dc3d0e5134368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/72119e365c6e39b7cf7ec8546c6a5c36da1655b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7480e7951511a7e38e5655cc8c05896b38a26535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/74f3c59b8d3627580daa826db103436df842c57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/767b03db2198dc6a5e85cc1b518985c4a510a05d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7aefb6852c9b2bde4c7c4355b72cca469302e6d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7b4ca96bfd0c37c627fd589c1bd59a78f7555f29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7cf6d16b60b5c3987431e448dec8d99e1f533c1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7d31f7c1bd8d2ebad276932bb4372c4399be78f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7fa43a4b7eee6335624bafd70aea9f96e5876831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/804171732e382c2ef3b1d96dafcb92efb1458611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/80f14c74c7cb22c37d352b1e16777a89d94f7bf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/82461c3eda042c5070f59ab5f6703e4b693db7b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/84039ad79ed29c8563fa868817c82fa2b8608b46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/84944aa5ea379d5b5f350d2a88ccad9575bd9ff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8602274cb1f008f878ebcf39bb905f454485d6e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8720b115f21ade879e8cbb1cac021de1b09e2a4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/890b98fb0689d139c7bb7505a1e4fc9cf37b61ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/896ef6d94bbe1aa62fd05ac517b1cfec47d058ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/89abfca3457c08f716f2d24b8ef5c30e9606958e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8abe63f3a1fe978a0a7118e6220d1bd0f4295308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8b7bec5d058a42909b45068f7ec4680a9cf14655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8bcca94d8804a899ba06d0f0f51c6092aa90adb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8cd3befb03b8805ea40e9013aac6f3671ca972b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8d164a8c8a07fdcdb5cec1a6987232c353dab729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8d9faabcef2c08fa63e41babe2a70deeade1370a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8fd4e615fb39cfcb9eadfe3f9374f954fd191b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/912e509902130f358243c122df1f41b336fa33b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/915bea5d7770efa7e910237e00eb64eca38ed940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/91ade420b0277ee38a2b6a7cab6163d2a1b21947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/91e66160c9c84c0fff59501afe6a229898b90940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/91f7e1ec52882339f635033aaa09fb3524c3231d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/93abfc132df720594f9333fd7af7eb9e68981503.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/93f100a34a402338f5b8da88f881d41013c4fef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/93fc24f7ace8f5f65ed3565445d17887e8c1e886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9413f4bc1fa3aeb99ee396212816870218c06fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/949e1ea8d193bc05af54a9bee84e8e02ca8d8f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/94dd95b6e0abb5e0755f17d5545162a7bf8945bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9705b8bdd11b966f84ecc7949a58a4a3afeeebe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/972e968b6dd2b8b4b37401266bd3d4bc53f444ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/978d51b5ebd979ce2e482b0bd5fbad724dc98588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/986a17e3ba93488550e8d69a52f6db0178e7fc8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/98a77c111669339ac0586349d7d56c5e464da25d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/98d58801981a0c614e163500ebb925829d95a54b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/99bc5cf27cfa3f78e7c1e60c7ac993db5b991017.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/99d398459df78557b50704cae9360a4e37c95392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9a07fef0ec520b0d208b39a84655b3308110c2ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9a5bb80f371742677c4a52e0f531217861a2242e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9ac4bf79e0e5152596b049f1515b5e37460a9057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9b66a2aad20b8da335442345e62882e13aca9504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9b9abd7ee797f88e558d7077d594c8ad38c3e9cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9c0a3017cf991fd6f36d4cac80cddda18f1f3a5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9c169a88557bee489f55a10c318ed473e8b2695d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9d7284675f7cdc29da0f0169d686b48675923888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9db1085707bb854afd5840fdd143ab3a14546af2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9e31d8538334f9d268d258e2f6d6fffd561d64e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a14f08ce904a26c49606aa41d6863cff13cf0f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a2a5a74248aaa4f86e2064cbbf940971da46f98d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a2ddb9a1230b14e10237b1c8d27ff7f50070d469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a2f09c29054322bbb92704be9e65e11a41e8817b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a3fc5bb7cc358dc0f94ce5cb0744b75104e75e68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a4ce00f984370b5fc0c0846bb7c33ace970a03c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a53639752a9686093c924bbf22d7ac6ab45c4d02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a5bf0648be1d6e1c0c2880c1f61fc13ff5b79127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a6d8ab9cca58bc4fec3d84211d40a7b0464852c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a7c6e0b3bdd366eca9aa62d2e63cca65ba7c1b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a7d770e061fa7ad848a62055cab8f5c554c8683c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a882affc397ba5f7cd00d95bb1a5b505c6926aec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a89e26fe7d356c7aeba67939248e6753b75975fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/aaaf35262bf88ebcb59eb4c353753724d024274f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ac7994636ab4edfe7e3f474f1ca09dae63b35a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ac87e340a86c74495e1ee70b884543e431d9405b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ad2b44648cc1c6222c37a4453759444c1235c7d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ad59ad42122d3c49027d7418f4d9c2c41619319b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ad87deccfcf0e42994378f865ea2b3f6c77376d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/af5bfeb37e872f923d52ec8ad649c713d2836a99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b01d154d04c2943589618ae771dead7756760f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b055bb6cee95d0f22be5c9737f3043c65ce98c0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b0c928e5a6e702bb4104d1bec4e7bb0f66508446.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b10560623d85f7ac14999613e738d2898598b1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b1ab8366b3e499ff84d7db96ca186d55331e600b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b1f196f59b5e9cde004a85fdb65c6697199c0079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b22d4415008f6318db620bbb7626e97c11093fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b3a41b4103b6a5eba13ff7c7f9a42a543ae47b51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b3d4eebb75d4b7398bf6be5a0cb35d3c4bde92f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b6a398c3b1bc6693c07e1e2dcb82d1e3912b5ecf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b6aec60a5c63f075cbfa236bc773d3c0a206df18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b9b7699c6ece6220350a184fcafdd8cfddb82e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ba022b2ae47832c437ab94d7dae6802381cfa4fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ba2587c046e3f279d0a2d8dd6a6a945b3af5859e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ba657cc3c1a83c048bbc7b017865b08b6d5cb7e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ba81cd0e70592b80813b8af864e071c763481c57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/bc077486e55f06a252837fecc7545b3c132d5e7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/bc4d7b986978545411f74c27613bb0d1edd8896b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/bc4ea3b80940c26e42f801c6cb7e585e05219e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/bd0f879ba31163e5199ba8eee9df156732bc93ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/bf3edf001593d1eab0eaca42e829c285cd166daf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c1b690a242c6e9e6b14e0b4d3e5914a76d744fb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c2333db2e8c6b7c70215f82d18ac9a4be2ce95e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c27a9e5d383c0d4be42f1410d59ae56b5344561b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c47aa7e7839e98741b0538fa23b0eff9ee67478d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c47fcdb603e9f85814f8eaa677cc12b545b3d57a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c516680a835870392c4465581bf53522a916d4bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c68f79f842b3b61ee7d42d98c4011d59e187166a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c70d26070b3b58d2dbd68df68799d93aed3d2547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c7e8a5518944aaa2de26876f3823971c0098c483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c809592f0084be5d895de6af5ce993c01939c23a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c8170842aeff59ff375b33c62e7f408e84ca1f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c8e8cb02446f49ce5755898f07bdf371809c959f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c91ce9b0c204a1941acd716064a9b160d99f4151.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/caeeafc4ed88547cd6ff69e68e4823207da04826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/cc74b1e28c0c1fddaac07b4e5b9258b96e174ed8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ccde4bfbc8a994f91ddbec3ef5c791588a102b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/cf856676f35cb44ac721707cdde54d42f63b3dd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d098a3235afa386c16e072d5461e1382fc1c3f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d1725671e52c9fef8a663b9d2d1df79e3e0eacc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d2f284db94581c3df3108f91c0ba3afb0a27efbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d3396c9f2e9ad51a4a450c38c513899c14b777ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d479330b984f8dd3eb59ac0707c0e52819e14433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d4a8f36227c6e457f75fd4f8401d1ff919f6ae79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d53a89087647e59af02d3648c26b09f36c7cf2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d55555e801f010a9fc2b881dfca91d6312f4405b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d5b9059023d73696e7ce07a55d007f6a00892b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d63261be758655f821dedab25014ce456d615979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d6b5d3f11cba223c365e51648fc2e4321261f560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d7a479a2d2e86b759ce311774a6332534df71903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d7bc67863fe1e8a6171ea75790e00ac61a695826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d7d271eba667f5278c5c4fd9ddbb5872cc341e8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d925074beef1cb0b5bc9ed481c569c15e4e29e38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/db61e4d0624e5927611d4790702c54d35e1be6df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/dc19779b6830281e6f5de2fa6b47b995a10577c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/dd9aa4b440ce27b4ea5b3101163d3028e8322d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/dee314ee02f228fa2dcd67db39960ede6e5f8839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/deffe9cdfe98fd7012affb452a775b11e5722c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/df94ec3a8cad1d576d52c1fa048976ecc756c8b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e074c5de5120b053f558b3bfe417a0ce8841a52e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e09fbae32634d42b88185525ef0ba9bc2d0879a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e1585070501a83a1e0b3826711c28a0aebe9aa5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e4a7182bea244d7f90ac5ccdfdccec7df9b2601a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e4aabd0ee66bfcdd55443414b83bcc49f6922095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e4bd11a4814ac142e79b889a1f33fc0a7ee950a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e50707a63e25c2137a91226cd4c9c4ccb503dd4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e66380cc519b4b7568972e584e19c83d5da77a1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e7d8b26c980c6a0e2fae8f287deff09b0e22619e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e80c31fdd1b9cec6f5dc755135243687886c919e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ea6a89ddbb1d9ab1e212b713937f949d04b5a0db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/eb28834c8afef82b307f20c6c14df0eefc95ebea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/eb99037e4feacaf029927a2b6ee31f4bb8536ef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ebd62c50660a6085ba6554046a186b0b4b03972b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ed695190479617f91f834c105684da4ececdc957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ed921e54b3c4267b0ff546ff091f99fd2b62c94a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ede4543fbeb598738b35a7b8d745e87702cc7f8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ee0ff601ef209af87dc2ecc4d23e32135c4555e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ee68a5b7160bb691a14340a5ea73d83d7473e525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ef97228923a17140b7c15239796cc641c47f0e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f0587e66a627cb77a4650469c23031b07c9003cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f13722b92905355f3ed3a13ba7404c51690bd26e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f23c4de9d52b9ae5eb56c1a7df53e405533362a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f3cba8973196d5b1233f070c2e8ca7b15ca7a3f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f4cc83ff5c0c7d83c4c94111e204ee03bf4bf236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f50affa53180950b0eb8cea11cc1f063ecf99ec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f863ffbf67892fedc76257494b3ac381b664e46a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f917f6a0fcee5fea3177672b19fa85afa125798a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/fa467037c1c6f9e6da62a0d58987b2c8e10604d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/fab43e3be7aed7b1856360994676519afe62950d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ff0f0ba57b81fadad915996b8ab966cdef4bf662.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2014-52/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2014-52/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2014-52/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2015-06.yaml b/configs/cccc/dedupe-url/CC-MAIN-2015-06.yaml new file mode 100644 index 00000000..e5f967e2 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2015-06.yaml @@ -0,0 +1,273 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/0754fa7ebacfbaf6492b30946ce01694a5139a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/07de10a03a03d6b7a34d664212d5df8f0d341c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/08fd245b75b07036d9fcdbc2e07bec91c8f50cfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/0a90927e8b9ab92d842cc884c01c5e8a76095a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855094.38/warc/709e9d200be5d6e5a9f0cc3a8f2e6b925d04d262.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855094.38/warc/e31cb8d7d6635dfe84cdd060f5d075d5da6d48c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855561.4/warc/7603df4d240467f0babf8b624d058479654f5d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855561.4/warc/cb4b8712f1592120aedcea7d0bbbb1b6d66feea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855845.27/warc/848f48d316ef5ccac80b299151e1814088bca967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855897.0/warc/4a01dc22f7380abe2e4978556678932239df4aa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855897.0/warc/8ec565486e66c4368e9e12e58eff84daa9ba4990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115856041.43/warc/49389ec6ca025c4848286e053972706158480384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115856087.17/warc/a0837f31ec0308f560afe74b7daff73595639c97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115856087.17/warc/f381dc78a11cd4a064a1020c8330ab5156bfd358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115856115.9/warc/a106b685f0b9dc01f7cdc6b831f6af1759c8f59c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115856115.9/warc/f47e4a190030903d4fa9554e75cc60a03a6056d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115857131.28/warc/8e9ce5c9dcaf3f84947ba99914c7265c5dcce9a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115857131.28/warc/a549e9b19a0bf19118212c6f02d11e509d3218ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115857200.13/warc/1cbc65f839b30ccdfd4a3e9b3570d180b0bcd7c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115857200.13/warc/876658c6b882c2b9755a87bafd5bee5ace7a4d76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858171.97/warc/1b65fe9ca66415aa1b9fa6f1bd323f8ccd8b726b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858171.97/warc/6edd4a782f921b39f6138b041ea2cfed59b490d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858171.97/warc/a69ef04a3e2045a8bb6c917d9b1ff92fdc3af3b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858580.32/warc/df912ad667e3e81d9a2d806fc5c4db0e5ef1d58b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858583.14/warc/06e130dc80fe4366b83ce3e3fb2e4e9c2ca3469a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858583.14/warc/de18fb1db73bedd24e983276e53ec1792f30e71d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858727.26/warc/0d58d79352b9aa854f4dc81accd55ecd1a258e69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858727.26/warc/2c8499b4bae2d637b19d9a0a91ef17627b84d154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115859923.61/warc/25b909110a33dcc93d06f111892ad01a68ace1c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115860277.59/warc/3903c3158e419e9302b16ea13eedcb941c1f243d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115860277.59/warc/87b590b1a04fa041c1dfa667ba63cf9ff96d42df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115860608.29/warc/b376fb67bc48d1ba3314c86e173070525396935d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861027.55/warc/0bfdd9ba670db4513291e824c02ec6a215a0509a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861027.55/warc/53a313199ba24694a0f1ca6933ccdaf3cb83fcbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861162.19/warc/b97a526ef608fab9b0350fbfddc3be51050b00c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861305.18/warc/385ff9a21d43d2d288d34bdb2c5105ff24073f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861305.18/warc/c5ab147fc3f913f413ec39ca4f9eddab9265b77a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861872.41/warc/204c5b8f740bf3bf55a29cef46454a4ad598c73a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861872.41/warc/b9f0bc5874cb7497295de75fef9253401ba50a06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862015.5/warc/ba55a3725a4cc69e8d9fa825d327813aa18506e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862141.23/warc/7b6e243ac8859fc9f7a9a92c1ed38bb68956e97e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862141.23/warc/d51a08295ba915e8176d8ac711220165af8d00f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862207.44/warc/071c8ad2f0065fcf79737122d00053352ff2ba2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862432.8/warc/23c590af347122f08e8723ab4cad2024cb5f4a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862432.8/warc/4f8f2434835b7f9f4f3b2b9436db9f1106ed968f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862441.40/warc/0a17faa92da588cbbfb7d0a1841d7c0efe7d7fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862636.1/warc/0fd5c7304e82a8e9e7c8c1d9378c2c9d9ef09f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862636.1/warc/d6a360bde7e8a62bb993d1edcc5762193631fc35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115863063.84/warc/e8cdeab94bc372c5604f9297d2997a2235fd1d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115863825.66/warc/bc45364a018948fcdcce61b337d14717c112d9b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115863825.66/warc/d301eee810e86a86289854e2ef3d0da5e122fec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115864313.15/warc/b52d650d5409f98e3a546d338b89b66780acd928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115865010.11/warc/64bf8406259d0825171459697004215109b306e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115865010.11/warc/71c2ec08d3745b6d408da5008e2727a2ddbeceea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115865430.52/warc/b21c2ac6c29aa2b4e8445b551d9103de7f02ff78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115867507.22/warc/422ba6b48aade8f891df281456bf42aa6b97eb2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115867507.22/warc/990baddaa7ed53d5161ebbe26d6010641afc8e25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115867691.21/warc/c3ae90ebdb80c1dadd8082aac4d90dc8a6ceed2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115868812.73/warc/62d0aa814bd6cdac74eebc15f1c54f8bc971d00e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115868812.73/warc/f50650863568c020449dc39f5ba9db74dabcdbd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869264.47/warc/1f61bcc3a12abb9790e012dede1515238f13402e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869264.47/warc/5b7f5c182cd422b5a0f48ea16d09b185169a9b28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869320.77/warc/f2fbc6093b8e5f222a6b4a480c7ef2aa168f61e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869320.77/warc/f6199e0f71f69d48c7882e4597d8a825ab05f626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869404.51/warc/4cfc57b7f8782abbf9124dcbc5eec722945434e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869404.51/warc/747875fe2630b97bc6d410366328dc2c11fdbac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869647.75/warc/0b53563cc4be80cc7270f18d6ed75e9293d8f80e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115891802.74/warc/13479c5303fac2eafc0a046c4f316dafea0be627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115891802.74/warc/c18559ea9dbe5aadbf4b5547e99c1a6051abef5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115899686.93/warc/9faef1db4136e7b09460d464e5414a6957c5e34c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115900160.86/warc/04318e8e960454414064b43a362c9c480ddecb9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115900160.86/warc/522c6d76d4ae7d298e25d73cb37f7db67a504c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115900471.63/warc/37d8c40ea20d16f6fad82fd68b40aad3d62784f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115926735.70/warc/0aa7576c2f956f98eac5022315d9eae2ea6b3077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115926735.70/warc/8d23937f5124bff020d8656a473436e28ba9b654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115926769.79/warc/e322e9c1bcd96008d80c881763be1c1023027ae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115927113.72/warc/e8cc98f6e10e2cb4bd8c902eaf6254b3c1d463cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115927113.72/warc/f1a055208c62b149452bf036619397d9caec654d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118059355.87/warc/888f62f226c7d3e757bbf9159b41717b05ea3940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118108509.48/warc/33d0829a292231ad48a38cedc485f26f2064c6d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118108509.48/warc/5c252f864896dce293407e71ec5381c8195dc9e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118551401.78/warc/295c9de66a911f11bf9bbac20b2435a44796953b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118888119.65/warc/56837eeb506d2e269d0b2c9621f69bfadbdd2fcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118888119.65/warc/8347f76a0d2e8b58ebb3af4dc62e5f4859b5b42e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118973352.69/warc/f38c8c17c2e2cc57d5783bc0a63d7ac5deda934a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422119446463.10/warc/3ebc743f140c8781dfd1dea6109dbadd8328b861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422119446463.10/warc/e844599e7be32d09f0ce6091db129085e8333e1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120394037.54/warc/48d9f6ab12c6de57417ea3255f5714e9491830dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120394037.54/warc/ae4539799a843841a47841fffb3999a01f542042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120453043.42/warc/c6dc4d17515362589e8682165e87668f73a901b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120842874.46/warc/02de22e84b13eda475c2467de9b796035af324d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120842874.46/warc/6a2a886d052536eb5abfbbfbf8ae3895cbeaef92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120928902.90/warc/ee6510c447dce3bc361694936528bc905bd199da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121478981.94/warc/3be166222fbe00f670f4fcfcc0177fe4a1644284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121478981.94/warc/cc8d4e19ea799bf62e2c8518c5e586d7b3efbcf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121540415.91/warc/dd2102da78111a67156b59c74b27aae6904ae6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121569156.58/warc/23e1ef2d8bbad26b35c81eb2da39422f7f27aeca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121569156.58/warc/c6327b11725bd69f6ece030b4253be3f99c140c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121744242.57/warc/d84914b121f3ee96f918b7c7f7925b133e46a73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121785385.35/warc/3f65492540cc3cd96adda9e56badc10e0051406e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121785385.35/warc/64ff4342dfbf31003b668e47f00fed8eac677333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121833101.33/warc/c5fbaf5ebf77f7199d543d7f37e23aee86d7df05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121899763.95/warc/169c48418a5999894cc91d360145227fff4fd35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121899763.95/warc/6eed14320d7d38bef7e877692e8a5928eebbd65b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121914832.36/warc/369c41b8aee14ce248007bd3bf56dd46dfe538a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121934081.85/warc/6ab99d2efb25cadb68613d8a062db856c09c260b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121934081.85/warc/e2776e30e027932fa19be263a034cd79f6a6781b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121961657.12/warc/b56123b8508795c8b780954af6391afaff9a863c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121967958.49/warc/544e922fbd3b6297bd5d35ef0d53a5fdd514c319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121967958.49/warc/f2ee451a5958f0ab3e1f0b03fe45d3cdd31c3bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121981339.16/warc/2f86c3ceb0f9c334d548381675b78958494b05a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121983086.76/warc/3498b65164b21d4d27eaba40aa5104b3cd892137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121983086.76/warc/8d675e64d096df19029e6b5191d86f7de17f1934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122022571.56/warc/2a9aa62a2fec9eab9be35b439337a6dbfc03a683.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122022571.56/warc/32ced2bffb7cb17a8e250ff35812f0736d131a53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122030742.53/warc/cbdf60f0daca7e69d1465c40643b487c5503ef2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122034298.3/warc/3b98d5873b7ba2177796da9cee6210dfbc6a3652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122034298.3/warc/a09ff606cdcda85e4d7fac007b6f43fea0a97a58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122039674.71/warc/14d09ae22b497594c993c50a61e94ce7b09dd6f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122047499.45/warc/1b7a112643793da50d7fd10ee121586708720619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122047499.45/warc/32a2671da6ff4dc154b51bb8ebe3a9ad11b4de67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122059136.7/warc/43f307e1c0eaa5e96e02db9778c249c45feea32b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122071449.31/warc/0d85193816402c88bf3147a8d26cdfa23296a735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122071449.31/warc/58d1ba07a42a8c1c5d0f2c5a168e329c6f700820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122080417.25/warc/d2042c961d0e952bab6d4eea89718d0652a0ab6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122086301.64/warc/04e0ece447ccec73edf1a71614b50ab5f7114688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122086301.64/warc/4ddaa9f5e1d886c5954306df3200ab567b76a1e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122086930.99/warc/186937236acd68b4fe904225e8f72abbabc4f5ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122087108.30/warc/4204933098d13841c11061d70bb8dd8b0154fdf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122087108.30/warc/9f3f3295563b93ce68a0319aaa6859a2f1c10da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122102237.39/warc/a18d1315bca90057504b9da9d05b368c216cc6d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122108378.68/warc/c4139f0ee8c8139eaaed8a476566baf72dc4da46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122108378.68/warc/d0ef7ef6c8a342df5dd77a9d86efaaa688631141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122122092.80/warc/4bd94b0f5483dbf3382d0ce7cb9762092bc461ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122123549.67/warc/15dec56b646e04d9dcadd68f67a67ef07a84fff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122123549.67/warc/6fa750bad222431599172b55925aa8ff302a82ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122127848.98/warc/627065eaa86e643f42f5add667571d7cb9b5280c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122152935.2/warc/6337a6b04718c34338103c0858c3868e394e18c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122152935.2/warc/aece40e2afe5679f962e4acb2d85236feff6da0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122166061.6/warc/2eaad5589714462b6f83467ccd9db962d396e426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122166061.6/warc/d947ceb07e06c582fdc577f1d59365062e807272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122189854.83/warc/0d6267b4221b6a4f629b72e18813f353263326db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122191023.37/warc/9d6c8a1b61827b93ebe169bf862ecdc46030caa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122191023.37/warc/e102eac4959924498a2d090d5d42625e2a2be093.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122192267.50/warc/60b4abc096c6260071295ed24f6d3708df3316fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122220909.62/warc/2474af3a4947e2e0b3221754c3bcba8287bb3f56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122220909.62/warc/bc8368aa44987f65981381ac5d9db40a73d7fd01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122222204.92/warc/bd9915cace1e0ff4dcafb09d0c62f46720e36c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122233086.24/warc/15b3b688a5d09ecfc16fa1b87707d0e230fc2f85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122233086.24/warc/b4dd27633553edd7ec28a16e516fe5962fd17b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122238667.96/warc/0984ab16b398c26cc98fbeb5ad1840ad2c8e6db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122238694.20/warc/7aa3d6ef49055b57b67bbd22daf6ce8b140c8f91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122238694.20/warc/b54991f440dae8f8855653068b6031b8e49d20db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122245449.81/warc/b2d7addc137d44aded144c2897d0fae5abb129f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122276676.89/warc/164681ee0e20669b537bbb478b92993ae93d4fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122276676.89/warc/a3ce7ad90dda061cc23b12ac58195bc647b7dd36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122328486.60/warc/95e9bc6d4121a807b00298209b392e7e532ab7a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122691893.34/warc/66fb4caa88af41a68d6d8a9b14b77753cab0adf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122691893.34/warc/889dd83a0c942b2fea3df20065c20035454f2b25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122691893.34/warc/bfa5358f7019f899fb5133655d4507e468dd215d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/158166c2b79e0aea60a64ca04b45faf67e0b0b98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/16370304e95359c6d6a0b9342e38d1908ae5f9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1a0209fe9120a2d688217a2206ab3648090fb118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1c3337d3cd3e2370df726f14f54d8731a638d21f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1c68b8bc52fa5e1831d2ef41673d1ffc987ba1e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/21654e3b65977b697d1381491ae6fa7934c84e1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/225ece17382eccbcd2429f65019084f72b5fc098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/22dd1a78b8b78767c28eb4fe6de0d05c19288f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/23dd0a7193c8e191d5ba3c5a400cb6e63870ab9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/277ac22c55938f7b041cd391e5057c4dff5ae02f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/278106f126a2a3b0f0ab9efde1a4ec81b80e1931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/28d28be653019476311c18cdefc7c00579004368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/2dad26e48a9841ee935722717be468d8b74b3fec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/2e7ff77efbbf2077e36d430ed2caf1379508f789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/2f267a0ef3b194def68a401491ae8efd1883d5da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/34000a5c6e6d160c23e9cb27ab3681f838dba00c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/36b471f74c3874ee946bdb122e51ed94df9cdc6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/406cd65dad8180c9b24006f10ab458cd453def47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/43656cdadbee9f76bf064e0454eac2273e00ec7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/49df37ec74bf838ee37a3c18fa1f74bb3f0999bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/4b5c2ddc36eae13f69012343353d4738321d5f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/4dc3f971f41c98bfea04e420925d2212af43ee32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/4e84b6aee538e486a151b3dc2a341f5956222ef5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/4ebf76a97e6fc4f02168119c5f940b3c36b39500.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5310c5d68b6216f0267ea2cd5ff63ad8f9cc7aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/55f3aae2dbc0876a0d78dc37ff96f366df9d96f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/564ed3da6f8cd4888923ff2d97c82e21befbc679.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5731267a1ba32c8c2bb854efaecb2a87ea4172aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/59bd6f20d6687932f718d8ba71cfd711fffa8978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/59cf3bdcd687ed617868edf498a3796762e75213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5b6e7ec0cab334ff0292c5db2ec4e827edae53ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5b99d64bc0173fec02965b193c99f1569c4948bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5ef39017b260cf8c9a584c1704e2bde4af84d61f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5efc2ded7211fa76e860fc8316dd9a53b4832191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/639fcf90deeb9bb2c3b4e3ca0cbc3b59f429234e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/666f0ae068ef4e2b3c263305ed50b6bdb413f64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/684c580ec9e0dd259ee3044cf1e65b391197d8ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/69472a513dc7bc8ab018ea7e1b008ce4604634f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/6a863513ebc732bb7e50992b0d3e9969bd00944a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/6afad92c0ca1dd297c845e77497abe94274d213a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/6d8cdedc5e6419e5ba65657eb505b1c942dda0d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/6dcb52e0530555eade421c6872e2a05a49f1a2a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/75d44835b7afc5d82bd22b7a273c3f1bc2fca668.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/7bff4920ba46c8a1e706e1a54b0b8708d9845e18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/7caa2cca421350dda82464a73e43b3b43853ef90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/801dd04b0179c2243f596e48a88e4bdf4a1a9da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/87594d5c507e95fe1a728f5fc0e5e99e70c83494.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/885702d2db68a0345969b139103b2d5f622f3e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/8ba78ecc2d5c3a80fc7d0d489a36483718c0e257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/8d757321184032682ff701352b03329e62739c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/8e59b7189b3c6093a11c738728aeb7fb1ca05f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/903680602c036a590d7f9713f02650737d2a56cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/9276e39a1d1167c147788311f5e7d79f3bdddc82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/92f1941b0f001f9afe0ea22995d45c4d623275c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/98667ecfb831faa99393f487ccd2894de701acd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/9b73748511be228de9f12ebdef2a1a838a8e44ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/a082cd68ace75d5ecbc9a9bdc6572c3d652287d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/a2e5e32c0bc113796fff572ea80754aff0e93c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/a4b708354159c7d9ed5c0b9ceb51b7bec691bc65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/acc2cf3ff61a877870b17a844e3d46fbb35da07c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/ae5f85e6fd21b337f4346f7ce70e9b6ff093c81e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/b3ee970980d9f108e9390330429c781fcf82db04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/b86eb119e35a562eb39de2642a3a32bb9fb70295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/ba0feb7f1f717aab91441cb4e0909b3939b5f161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/ba3e29398200f664e82ba4677d676ad08ea24e63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/bcabe1b4626a936282fa773e96033ee6ddeb7951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/bf44dd8b2586f26138521abfe5056df82834d53f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/bf72234a557b717fedab761db39b71e6b4119297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/c4a9967a6c717a4d7e5d8549992e5049a5cff336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/c4efb7f3f3bc3977f763c937200b7f6101f7010c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/c541f330048050fd45ee0b64526ce187a0fd2bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/c683b1856afef520c03410bc27e59c2421de5905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/c9a36998739422f58a3adc8a3582641e26429cf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/cd66de1c179369476592f4b71313d4b80291cebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/cf96ce91694bfd8be393ed1cf67a0e5c5c808ffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/d21f24dda585b065db7e7f61c3a120a2156d0ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/d52525db3d3920f70a368349ab184550039aee43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/d6f6891ff56a482167505de44e9eedd82281fae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/db711696d055ec0fc4e5134476651d4ad75b160f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/dc69088b669f14d604953f3da38e11d233577ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/e0850ed56173f298543ee11713d78967fa1b90af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/e71330d8bac08d1308536e5544de4a3c4cd84b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/e862c9a67cf74bd64580c7ba40f81debc3500568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/e93350a857cd7ceb1c843be74b1f9f39ca3acfa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/e9620adc7ea5a7fd04968b1f563cea6bcee0c380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/eaf0e4b0ac2f1b5428425a15578afbe61bb19921.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/ed9ee2aedc99309179fe63766cbe6c5796fcfc39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/ef4ff2049045dad73395214de7fddd9fec0c48e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/f29a642c9a6f32a3ccc93e632beb43243f534fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/f5961330872024f2040391747577d8db92e6c9b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/f76851eec04cb2ee642469e2bec0796816a2bad8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/fe39b38798350e8a582a632c3cb938ff62b5d522.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2015-06/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2015-06/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2015-06/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2015-11.yaml b/configs/cccc/dedupe-url/CC-MAIN-2015-11.yaml new file mode 100644 index 00000000..e9a5b443 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2015-11.yaml @@ -0,0 +1,352 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/07f33c0e981cdf12e50fabf7480a47a8b61ce627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/0e6891a82aa3ae1072d88e6978403f6aa29d9fb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/119be949c2691fbcb0d070e45947980169de9d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1206ed7b98e73dd1f98611c18a49f3ba15710447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936459277.13/warc/654ebcdc3014f7c83e024a50a752d6da01ef1f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936459277.13/warc/86d7275f80cedeb0ee3075e9025ff03e3da0499e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936459277.13/warc/ae94287bec2c59d6ab16683e97b414a6337f4353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936459513.8/warc/04e22020fdb184ccffe3d066d477ad35b5a913d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936459513.8/warc/bc1cde275aee99e9b0d35bf7276149892862a515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460472.17/warc/9e1a744786b38ec6e8afb3c7e346c01127fc710c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460472.17/warc/a4cad0cb407a2ed3ab8e959150d65df1c0237c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460576.24/warc/6292b03f2d1de6be0fda3bb2baf5e1e564bc7457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460576.24/warc/84d1c324c3b0eaa0d720a234e6bbfc419d593f0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460576.24/warc/f1123f634458a1ce04847c32e89f165a2441f5e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460577.67/warc/bde02037a27ccf4950fbb2c7939b8e44358e6d80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460577.67/warc/ca81aeaec6fb8609cf532abf0db482ab1a58c417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460839.35/warc/2e34a5d322f5cbf218c9cdc80a867896e14a3ea7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460839.35/warc/518acae9593a9cefe3493b318c132d09d145d878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461216.38/warc/1fc03a4bd7aaec1c3b7a6d39cf553e1794895d8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461216.38/warc/2dc1cdedf30fd043e9f1432d01c7150f763f1283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461216.38/warc/8c4cba589631ab02a9c756c84d448b1f570b1abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461266.22/warc/32bfe33819e778bec10feb8a94e696e29e0ad6db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461266.22/warc/781e9f97103948fddfcedd801c80756b40943e94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461332.16/warc/dcc2feef79508ffba38a90f9f77d96d03cbf26c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461332.16/warc/fc8ebbb2cf222c6475c4045567a342d13fe2917f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461359.90/warc/e1b95d1414b596b0b0b7292001212901b5d07570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461359.90/warc/eca27bc10b00777f1e365b2193e906ef5f339917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461416.42/warc/4040d8b420ded897f01755103587c87be310a716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461416.42/warc/6b590531b9c87f2b6766f948b9e6634c0291559e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461416.42/warc/a266110415ff482bc54614a21c380f2d6c1b9178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461494.41/warc/2bcaecffb1057f4de6972fd3f18b890614b0c252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461494.41/warc/ad4228aa4c087ce76aadaf1d55de1fa604fb76c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461612.31/warc/a1f68f20075c34ae56a30ae59a1d9309ddcf18e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461612.31/warc/ce2bad2515c1d3893883cd762ad34a8526fae633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461647.44/warc/13bf33672daaf652986d2ba05766bb808f2bcdaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461647.44/warc/b9e8a3c58386c5c3bbd335ee32b86c4014f4f100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461650.2/warc/0e069a37d7303ed50f0051f8e4753946ecf6c4b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461650.2/warc/cdf4818be9ec2329d8365e4c4ba7a5c6388f90ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461650.2/warc/efe38f730d037f588a355588858f2b2a21ab5d53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461848.26/warc/0f1e12c094128dbb0d0f0812799c3f0064dde74c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461848.26/warc/4a92c96607877590961c72d91529134cc6707fb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461907.65/warc/598060e9d4b2f6f31eca024f87cbfbe25eec8237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461907.65/warc/6dbc1c031e2827bd48a143cdeddf5c6d0763d2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461944.75/warc/01002110a7fb426839bd180ecbc0b8849e227ca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461944.75/warc/ca2d85de20c3297241fa39cede8e5106f0e82d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461988.0/warc/1dd94ee3911a79b0d2374ec32ec087a3921c633e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461988.0/warc/24dc1ebac9ad388853592afd572ef61b8cbef482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461988.0/warc/d5afc8783d70ad3915e1708df80521a53f2cb459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461995.64/warc/2e950cb6cbb20cbe634ce4c059ccd330def74994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461995.64/warc/5eb19377fdcf56632aeba171a4ac400038dd408e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462009.45/warc/92cb73b4b7d3544001cd496d505e9f7fe748c6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462009.45/warc/b99bd4127208eaa8f0f79e7c2a944fe7855dfe1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462035.86/warc/9d0b92dba6ad173549161791212505fc9ba39fc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462035.86/warc/c43c4a2836fb130d5af7516450d45ba3e167a342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462070.88/warc/1242a799ed693d8b19b6c096e5f4f2cef5a4236b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462070.88/warc/4565bb701587aa103b221810a3147431afd8474c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462070.88/warc/96194bdce1598c0d0a8792d654b73d11afbb3fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462099.15/warc/cdf2d9c521ad01c54dcf5e97fc930fd105da0055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462099.15/warc/e95d0bc44a3b49d04c2fbc6b21a84e86ea70c344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462141.68/warc/985a454f812fa72b9d8bac62eb99a6b097ae2b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462141.68/warc/d8ed6bfd062f12fa0013226f18008e45ab9621a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462202.69/warc/914a3901af580695150eba56a46ac11a23447df5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462202.69/warc/99cf101e2dce064af4330467deafb821c3f9e681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462202.69/warc/e59c1572d16896a86146326b41ac559ddc05cc14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462206.12/warc/38fa256787b0850175e4996658ace16453a2ad08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462206.12/warc/d7e730223edd9f6c2283b42e3272359cd341074b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462232.5/warc/0b9885a41f01b8079a5a0b1596a7d1641cc5668b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462232.5/warc/43bb8cc1c0cfda3404d9db3e5c656a366ccf69d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462313.6/warc/60d760a939aaf1d45c4eb45e2ddbc47888e31b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462313.6/warc/65da22589b92362c40d49d059f0090fff2986102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462316.51/warc/0d458ace64d8b73c9376ed13bf068fd7a485e31b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462316.51/warc/b72b8bc91d62b2ba0625141cf8c2ee8ee538c50f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462316.51/warc/bb71bf38eac41425980a413cc6d58f5fcb883860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462331.30/warc/4e11017500f35c136380e8188c88704eaaed0ea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462331.30/warc/6a465352bb7bc61347bd95c3f1117d7384525091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462426.4/warc/086e63727e880dd695298d4f2110a8add64bc585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462426.4/warc/9aa36cd49a2540a0fc9844b5ae8809fe4a1fd61d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462472.19/warc/09fa9b39d74db01b20dc81cec056f23608c3b392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462472.19/warc/0c75ab582889e3c889a681e14e9e1bb20f492aa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462548.53/warc/79f09e062594382264d05050789b7a6748d9371f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462548.53/warc/b1bf3fd600707905de34e2490f867b6ccc8c5906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462548.53/warc/f8c825006b55e6c08bcb6b1b55905932ca0614b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462555.21/warc/9c14529fd2129eb16c7e20cb05cba0f12c6bf5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462555.21/warc/b0de95811b7f98139afa04bbcb579fb2dc556bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462573.55/warc/8ca9dc0dff29effd5d3446850ce18053a910feed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462573.55/warc/fdb322e7cf0c965c7ba9e3202530ee617856406b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462577.56/warc/216c5413c9784ee2bd3c447fe32849b85b4a5aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462577.56/warc/794f1d4a57be99b7d68a2c8db4dcbfece49db901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462577.56/warc/cadad4cb17c330d6cb042e98f773951677d9672d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462700.28/warc/141110ed0023fcdb0eac011101c60051a8120a8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462700.28/warc/6f71b814581663a747b415f80d829a63261f2414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462710.27/warc/5d764b5e51ba91259284918ceb253401e070bb7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462710.27/warc/9aef5f1069f6b77353cd8166bcab60e37c862f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462720.36/warc/ea790a4d913dd490191f162fadefdf5f2d7d0946.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462720.36/warc/f3aebd84659aa09ccb8621f59b8bb6c93765194c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462720.36/warc/f8dae3afb2b5094198021f334ccfb37de2f66c73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462751.83/warc/31ea7de4cc4a86e2bd2062a9d681fb15f0b89b0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462751.83/warc/8a2755a43a0b8c503dd2814b9788ce7641da1682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462751.85/warc/44b5d1d631c9ba4a8af528200e461b4bd00fb48b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462751.85/warc/91e42ac5b8cd1e27e1ca96c1037fec0d168542ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462762.87/warc/3bb9ba8d2eccb04c1408191741c1f55568f1ea05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462762.87/warc/b9ffe42ef18a03112d676e51f00d02ffc4fa7627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462839.81/warc/a986d5c55f1bd4e7f219067d48ae97ce5a7c9643.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462839.81/warc/b19c6b18e5dcc544ec03c4187a0c722ce1041ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462839.81/warc/cd8196148ffe588c219c73740becd987d10a2248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462898.92/warc/2a082eafc6849f885905b13c23687ad416af4b29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462898.92/warc/aa5e29b3f4b9da6177362e94300d7c277cd282ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462982.10/warc/321229ad2030c3947f4cd3dff99a4d5f70a8b1ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462982.10/warc/676e2ca9b591ce0083dd1361c5fa5f0b3f85a7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463028.70/warc/4b426dd7949e6c248ade6dfc69b5ac4dca77145b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463028.70/warc/7f49342b6770d526d5603a1e4fac4486d88ad8fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463028.70/warc/f3baa5cefd720430532e1f47135944fa446a9550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463093.76/warc/045617e3e5761c8af3209fd1340edf11e716bcee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463093.76/warc/89ea896cdd8eebe6e263fa4bf28e5d043d9fe531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463103.84/warc/0100cf8189498ce59cc5afab873543483af0bbbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463103.84/warc/bece6959552e3fa0c38ff3d924acae626ca96428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463104.7/warc/1ea7f3c7b2e8c98ad9ec5597ae7fdff1457f9286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463104.7/warc/25bba774cdbee6bbf3fc33411af08f8a0f7f7682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463108.89/warc/413e833daaf96917dd1dea019659f33072e8acd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463108.89/warc/57553bffec955e9a508edc8702c4bf7a02be54ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463108.89/warc/a73e585b164a98158e752cc333eaa50cee19146c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463122.1/warc/030bb3c88c1f1bd971536d11b9b16b9188bb77da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463122.1/warc/68aa0b8fc8fa5c5ac75989a162b4b6d437d7b279.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463165.18/warc/d95a01302ae9eae5fb4e1b958e9eb051d8090aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463165.18/warc/e703db8d27d96d92c65cee817bb94226924dc127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463181.39/warc/235373f77c43b7f4c4eb867d03a1adc8eb682ebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463181.39/warc/5256b118dcac3349390078d8d3dec5182d81b32f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463287.91/warc/211860ee26410e48d3dd8a6cb04e540c6b3a7b95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463287.91/warc/792fbcc0dc4b28515609388baab60da3410f3376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463287.91/warc/b533735445c7d1a9b9b26af199ad33127234c376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463318.72/warc/7a6ce94837b25291039ec783d87ff106025b87fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463318.72/warc/f3f8c80ae80545d79ac8daf58271d90444a1b888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463340.29/warc/818e3e274e0e0b745101af34003f3861d4e03964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463340.29/warc/c93f226980354c9c1760b451e401d2b6d4be6f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463378.98/warc/901e2783175479004b6e8c9802736fa00f996dbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463378.98/warc/e833a173411bb3a7279861cfdad7ae279c55caec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463411.14/warc/029396f998695b832260c0c02bfb110f4a99fe0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463411.14/warc/8fae9684167121d6b380a28efc5f8b1a8e16f3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463411.14/warc/da886be41b5f7452ef80cf63a9e968d80a1413d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463420.52/warc/c29ea907946ce3d89273a5e456f0410129b7775e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463420.52/warc/ebc71637d03392653e9df9e3a3926a9fd3be77a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463425.63/warc/53f7e5595380347a823a70b826d4a4e5df16c410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463425.63/warc/be8524904296499138bb34ffa5a09fa717156e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463444.94/warc/0414ef0561b94a663da3c84cdc8add06a581e4e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463444.94/warc/08ceedc05e6f054e929e9e2295544d98335269c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463444.94/warc/a7fffb118ce53bd9236864825fcaba8ac9d8dc86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463453.54/warc/906bbc1b2ceccc99906dc9cc6acd78a48339b83d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463453.54/warc/e0c04eb5e50b11936fd20057ba320da3d4ea32e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463458.93/warc/7895c6ccf7216b575b9f6a450a49157d8f366d2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463458.93/warc/95d7277623f959a0c57a1d7df779fb7e648c0cc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463460.96/warc/8a090663719bb39c38a4902ef1d97d38783905bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463460.96/warc/b8507fb91ae50ec927bf04ba864498cccf973f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463475.57/warc/14277f017132f0a1135968da0ca441eb9c69b34d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463475.57/warc/914fee06a323eb0a38a620f538fb87992b528d08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463475.57/warc/e07e1e9b2d23527c4b32ed7408849f51688f784e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463485.78/warc/013a0246e51ae9377854b6b12b61e9477b675335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463485.78/warc/d492eff3b0c70bfb9bd1183a92191fa48dac9da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463606.79/warc/64e3d691186539f74704030e90e9f9283a2f29c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463606.79/warc/dbad538463f493bceb6c219c528e37876f7bc120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463608.25/warc/37a7235fe72b77935903dff09972273012f21240.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463608.25/warc/b0bc695f474ae54ff17a42a6badfe98250403458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463608.25/warc/b87c376ccbd45a6ebbc4ce8f2de5a21440a9b527.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463637.80/warc/b6992cca78fa0793b5174cb0fe79b0abb5dada84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463637.80/warc/be2a6921b173abada9d1d3db024efd19e841c99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463658.66/warc/a60d57492f6716ff917fa32d17ea4230778b9de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463658.66/warc/e72a6d674a6540574d6af67ae0700fe63c282e4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463660.11/warc/2738ab0cd0bf9cc7c535bfc49bb2f98a1691f1ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463660.11/warc/7953f5edc54f84a1b795714e49b4b1362c9d2193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463676.59/warc/118b4f5fd8dcdf954ad32e71be78ca0e5223b78d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463676.59/warc/c99e8cbc3276b7496db2b0a911ccd7014a3b7449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463676.59/warc/f521bd22f1ae2047466800d206396bc849d30781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463679.20/warc/274be3b9d1466c776b15a1372bcbbc390dfc659d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463679.20/warc/78681bc672f97c6b1602e0ba5c987e1d387b1d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463708.99/warc/30873d99e2870c066f7eff7a1fb36891f9124dfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463708.99/warc/536fd2435ba003f701bdc00a98fee97520276b6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463708.99/warc/77324c4c3d999caa9feccc1bd74de61b118c526c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463928.32/warc/00144b2da0c7bc6f65945bbbb357f98e745da1bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463928.32/warc/4bf3e7d708e29263ef2a812714086639de2a03aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463956.95/warc/5862cfdcf7498eb1f486345dffd3715e403199a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463956.95/warc/ac4614c0044751f3ed7ee834ae80d21840ccd6a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464088.46/warc/0c4f1b43d9c17c86e8233b0b9a73d609fba04dee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464088.46/warc/3af5ae8d8424dcab52af0ddc105bea298d2549ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464088.46/warc/8f741c6c7d239a5eed408738b90291bcf05278fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464123.82/warc/11e79d7965ad921f6c8599087149084b8bd5e2c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464123.82/warc/68c4ffe7cdaa92bd272d9d64768a76f3f8075586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464193.61/warc/77be26d4852c0b8b05747f8dcc8769e810d54c88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464193.61/warc/8a4e4cd6bd2731bc3f1eee17f3c9cc32c01ded69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464303.77/warc/036091895325c1da935111b3224b7baeefa3a7db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464303.77/warc/86672751fb33665de6691c10cbae30d7d4271aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464303.77/warc/ecdad5e1270fb5f8aff1601cf2bf33a7889b718c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464809.62/warc/b8dca6e433b12938bbd3d4f3d2e0c8a690bd7c71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464809.62/warc/e281644724ab7cf6423e7d947a7975b835a9ce85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464840.47/warc/a52aad5e449317a038ed2884ddb72c9931482b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464840.47/warc/c71550020b6388fb2685e29780bb40cbb8c2f362.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464876.43/warc/64b05ee5a1fbfb0ef2e59c5833878a118f4ba44e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464876.43/warc/dee72a889b0baab86df564f7bad8d3f266eef405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465069.3/warc/68d3d51ae5ee1fa6f2a91041b925da294f694a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465069.3/warc/7a67ed3c745b8f2ef83e5277dd20f6e4807eabd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465069.3/warc/939ff3ff6cb4c6594ebe025ad43d1763b92ac77a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465456.40/warc/420b559450d4ae7b071110e006fc8adef658ef4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465456.40/warc/aa5a619339be4191eafb15067db93a49a911f816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465487.60/warc/54730d29d062a0aa280472628a15a0e4f2aee35b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465487.60/warc/7dd339394936b69a19b1f454868d453557f549a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465487.60/warc/d47c0177a93b2277a5e1cddd31eb9db6ea6cec26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465599.34/warc/4b54d0b8d4814a4f2d0e0dec1c797ff336c6a8f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465599.34/warc/eb9a244b687251890d4124e7bd14a921bc8f5e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465693.74/warc/792b5bc3521901cc5ce2023e0337861fda65b312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465693.74/warc/e742e502c25e7fb40026deb9c6a182a6efc52e3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936466999.23/warc/abe7e1d38f679531d7d82f954f458cfa6f40af56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936466999.23/warc/d1806282d94a3d4fd2bd625136ec9f79862a8c9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936466999.23/warc/f13b38a6189ecf028cdb94bebc823e6ad4d907a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936468536.58/warc/c00b102b51a5f6df403a92871ca374be2069e827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936468536.58/warc/ce439eddabc5a854f5d7584e4e348548be1106ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936468546.71/warc/0c7a2c1c71809377963eafd8bf95340f67a9bc58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936468546.71/warc/e0cc3460d8020578eda272f33378693c1e6af78a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469016.33/warc/a36991130991cb2ced3c98a83ca13c829df84716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469016.33/warc/cf3190575fefefb1a6891d771d7d77fe7b88b352.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469016.33/warc/d8ea49c950a7b95b4402df611c025a5a48af587f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469077.97/warc/566d2623ba31c1e88fc4eb397e8567b524f97288.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469077.97/warc/efdf63fcce611327e79827251d4c586e47598a42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469305.48/warc/5fec88c315bfafb1dcc045bdbf609c6811299000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469305.48/warc/c1b53df30e8dea034623c7d2d9b853db48295ec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469305.48/warc/efc32fd3c3263354657e8d97ef76b77c6e848c22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936470419.73/warc/85766fb022e9739382e1c4b3a7cbd4e2e821f5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936470419.73/warc/d397dd9c8e56c66686869d947b240ec0515f7df0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936471203.9/warc/7a0ae1898591c8b62bb7519bd7fbd7b8acf3c152.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936471203.9/warc/b30191fff83ae622d1cf59eb6ffcfdafa5152aaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936535306.37/warc/418555cf0bad7d825b53cb9c6f3bab264c46195c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936535306.37/warc/6703b5326ac6cbae0cfcaf76527744ed08816496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936535306.37/warc/b8227e3633aaf9c968cf7bd35c02c538d041455e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937406179.50/warc/73c4d83b498f6d85b17d32d9cd1b92f7087220c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937406179.50/warc/eec23cbfb1051ab340be44e342cbce8f10717fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937481488.49/warc/5ce89eb20760553427ed41bd10e3fcb31eeab6df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937481488.49/warc/7eff2cfd1961023857193939b7ff647f41403f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937481488.49/warc/c3ec385aca2581c7d7b2a5a3f10955f297d3ca57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937481488.49/warc/f848de25e3857b0c34b0b7fecf07bf5d86a010b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1484d5acb306a7a3e41344466f1e33db37cb0180.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1733a42be3e5c3b17cc275e6ec87a8b1cb5c4473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1e52d5f604ac5dc474f58459f93aba029c0eb309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/2490010ebdda0def09449adc40b9c059ca1978b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/2dab6f794648bc137dd6db3389e33605e4ad49b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/2ea3881e6a7d47f4c23af6547ebdda62279bf3dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/2ebf8867ab8c6e99d5a390e839628ce225db4fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/2f6f9221c9ef077ecb075eb45eb2c85e328f3724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/32c7f774efa693a74645931135ad57fb837bf08b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/33f79b6b36a781984808fb525369c0db0afcfee5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/353464708466879f5221dd82785c9d973ce0daf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/36c94e6a650df654aac2a09732718a279875ff2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/3743824633fb2d953b69f8ba8e36f56e670f3ad3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/3b5ef96684f573e31cedf1333fc8d77421d29443.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/3c4f8fa0d3b953ecdc83cb8c54477a8a6d6c39d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/3db421f58626adfeb369db683808f05d1a5dd544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/4492d07527ff819a1242408cda7968f328c57b43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/451f98ae7b08c7774409e5f97f5ffa026aa41809.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/468f62d4b5cde760315d20883bbbf7bb12dea621.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/4818d16ec7d59cc4f63dff97db2230d774a627ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/48e5cfad77fc28ef994f308c0d74803c8b95061a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/4dc4d192d9413d47e16dbc395a4c54c78413086f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/54591b38e0d0b9e2c41d0076b6cd498201622439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/557e00e19f79b440463ed1015af4e3c519b896ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/565995c0dc0c4ce1bb83690c6d4101642471d19f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/56ac782f1f2adac94a8a3c743192e24c94bc4891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/5a69c605f86bf25b34e82007139924c48e524518.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/5c1be8042cf8220984f0123d99bf67a298bcc67e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/5cc81e994da63948f3f28d4f7c31b71cba0422ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/602faf76cfd190b857d2f7bdbdc40c6d24d1f9c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6703f6c468a1f502f42cd6edf965bb060cf71210.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/674219cc86b959c8fd8baf4fddeab2a9f26b5bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6977e2d188330d2a58d36a01c214230b5047671a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/69c25539737c048d78745fd3b59a3e5ac456374f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6d0904adff9aa0248641be23ff69ec4681257a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6f41c51325d8d9c5e232e980dff8adbfb24091e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6f7ed89fabd22a7846cc5bc9f6dafe7a46ebbe40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6fc2c11430f9401c37317557bb87ed909ba6fa3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7060a6d73c38d75eb744d262f33c108f88803695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/75160be968c39f58b375d8881d8bf145af09a4a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/77075ffa29887c8dc6129b574d1932538ccb8326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/776dc9024181adf052a026659372bb2614c22bee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/78dd9ebafa48134cb0fb633028113cf3f0b5657f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7a0dc656a75c3432ddbd9dcd6a63d8f16b88d110.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7afcffe881ab4d0f11fdbfbc364f9e8231bfb0f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7beebbedc452e0f196955e6358c596856ab254bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7c999d343ab073aac1ee6fbaf61df1ff0ef675c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7cb6da2ce060256ba8bce1b3c94c35cd73d8cd7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7dbcdd2caebc7b7d054d2bf31c9ae7f24935661f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/806efff623848f5451174cf1af322bd94b81d485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/813a702ea86ab5ed29a427c04b8cf49c6294dfb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/821e9cca2735a2a4b2b37911b507e1e2cf6fbf3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/8f8b647598fd96a36c8991fadb3f8df374b70b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/8fbabf4452141a5e77cae69bac858e42feecf26f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/8fc25dd390d0ab2af6c864222a173695f804546b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/97ec298f754d752ca329f7d67dee3293a6ebf40b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/9930ba80b193d80af4c804a7eba86d11d3a160b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/9aed8c461b607defdd26dea7efa942b8fc6cb59f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/9e0781617c293e9d12edac54d71440364f94b8aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/a0f2f88eceaa4be6ae7acfaf8ef644085493f216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/a2cd35410045bed7707feee7f9a29358e2584bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/a5d7f2e8d763d76c5c6f7e5e89bcbe1f308f28b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/a67c0b9981372111ffa0f24410a6ae662f9955bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/a6f73149fe42e273c2f306c470fde24f53a936cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/aa0b7812d30277cd2f48fcf0d7403eb94f6c15fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/acf6ed2194ee92f40eb03d3b6385630c2c4ded98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/ae6ac7735d7d4a08c0484bdefe4d96b2c57b1912.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/c3e94e9552915232e2368f09eb97e9b8269365e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/c45eb98e3dfd12aac597e798920d74426567abf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/c59d0d62033dae31133e806d44540fd18e90f577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/cbc3773e18a409a0f81130d47a9d5ee737e74e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/cd5f452d94047201ea6e129e08d1cee0f706b69c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/d41bf464d81786c4db2461348e0ba685cd62913e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/d6e4d53810e6b41aa6b581d7edc186114e2f347a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/d92924487fc9b14f0e81c81fa4a47cef73e96caa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/db4cec80d48b123a3240c4e206e06fd046f06219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/db91bf3f475ebb9250a61109ce4f11d60c992c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/ddf3aaa22db732fa505ab16a8acdf783eb2ad322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e1afaa56e7f13c0bb39e7e9e201a01fcc4d69d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e1e6b3d1aec82894c38ac075ac17955f04aaaff7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e3d7893835dd751d1098cc7da199d40a2bfa8d63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e4fa4a89767acde4e80cfaf0a94b54e8d8e44b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e7ad059a990f348c3a7ecd5a3d7de3d8dc5a65c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e7cd11f2da8ba03949a0bb7f389037a2da3598a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/ed574c7b88509c852d660783e47b4ff02286017b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/ed72635ac32138eede92cf9850da3a13fa531bc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/edd0192048fe791f9121b1f3062cb9ad257e284b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/ef54e41085dbb7b0e254ad839cd700db38fb1bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/f019a4c4105206f7dd464bb552a48954a10ef207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/f262ec11d89e6f354740a0aed5dc87ad4cede42f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/f28354e140b363ce7c3762743fb97f271dbaa887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/f376be0299d7ae030076cb66521f6a2d1d02b2f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/f8b8efd2169565001b7cdbd760a0a17399e29452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/fe80673a8823ca18eb83273393d0ce429adc2d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/fff0b11da4ae3e204ae44a5e893c8db22aa46324.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2015-11/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2015-11/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2015-11/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2015-14.yaml b/configs/cccc/dedupe-url/CC-MAIN-2015-14.yaml new file mode 100644 index 00000000..34534ddd --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2015-14.yaml @@ -0,0 +1,313 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/028e1e27dfd6df05b1c15db79a62656cf5fc9982.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0b038ab2a431465ef55d52b5bf7da3b2cdb4a2eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0b9adbc764e7496c2b456137a7b2e0882deca598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0d0571abe91ea639c6e870059d8b968402827b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0d0971a9990ee3b9c7920f3fb5cb54d2c7f608b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0e5f7efd18c9351753091eb5be5ba6533c3fb45d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0eaab183ec416e4fe9dc941aef455c1dc1c7a0d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0eae862dcfc3b80452c38c7279451dd85366ba51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/13df18afc722048d37b65b1b1fc8590fb8cb4eff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292567.7/warc/7b6a8b145d0476f8fd57c668b1d7b9fd030bce69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292567.7/warc/8f13b7a279342e2f688a644bb0c26b7537e4156d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292621.16/warc/6298d25923155ca42464760040e5b2e8a84b6b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292621.16/warc/eb69a1a711a0e1abefad9f147068c6dc282552d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292683.3/warc/355957638360d6dbf7d5d6082a2815d5afc095eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292683.3/warc/6e7c8fb46244157917a6ea10e2e3011d8d8068e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131293283.10/warc/2007f504c3f03964bd8eb8473bf5e48c3597e15b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131293283.10/warc/9a268d410651850780924622e732db612a8564f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131293580.17/warc/392ad3df248ad383b257ee26c9d44e36432d6c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131293580.17/warc/646fd610f3211e35acfbe85d101e9af24cbe3545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131294307.1/warc/c9e9f70cc692ff90e288441c18d4bacb78f97560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131294307.1/warc/e5bd64ba42f16569a0505504a971c5f9ec3be64c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295084.53/warc/0759aed50b9cd34d71e3afeb412c400f09f45cfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295084.53/warc/92dde3410102b6caf127634a697a4222defebb1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295619.33/warc/941a5d251342d6730d563c445c60826663bbfe77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295619.33/warc/cb5fc875cd4eb45b9c080a8e76218fdbe7e48954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295993.24/warc/0d93b94bc77ce3b563959a163f8ce73357289101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295993.24/warc/af3aef701c320a826a4d3e30ac3fe9feb74f08db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296169.46/warc/68a1e80023b5ce1cdbb2cfca47736d478fc6acd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296169.46/warc/f6469c41ea78e799b83c3c93c37a69b741cd9da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296383.42/warc/4ad15508c65f43a9ec7691f8da9d8711ec6658ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296383.42/warc/be82defe37921c6f77971c61209e878e72a05512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296456.82/warc/719cad13a17fd1634d1b5dd70327addfeac7ad71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296456.82/warc/e3c4469b3c8292fc91cb19710359e607eb7ed057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296462.22/warc/5c8b5b1b4ad611bb0da4a5fe6bc2a3c9ffc6c176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296587.89/warc/4dbf36a11af58308e8e50f4b26e015da5d1c418d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296587.89/warc/d70a7d9a3f6cb6ab67f6424cc5a8dd92200ef569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296603.6/warc/96d1507542aba5715f0a16601848af18838fc4c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296603.6/warc/ea1fcc9f9e8444f20e9e58a58cf3d8b414dff8e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296951.54/warc/90d4c6a3b591baea9b5cab01b13d2bdba295110a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296951.54/warc/ede53a471c9ffc349202f006db435c2c02374dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296974.26/warc/49956c514552b5ce69e5c3ca219c26043788de73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296974.26/warc/6cff16eaed2cd02d9259c6d6c40c44c4e88fb9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297146.11/warc/0c3c9fc7ad15620e2a5114fa274c5c4fdb11da3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297146.11/warc/b45cc31053c20699c89349926eb59f32e7f3a849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297172.60/warc/13940d4361478556e458989836fbc7925229c570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297172.60/warc/fb7293186e5a863e23e242c706b07c00da94b898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297195.79/warc/28ff06e67455f40396ad5ef0c145244cc6940dcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297195.79/warc/558754dbcc6813d50e42974968dda5a64cca2802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297281.13/warc/4dcdcaa15be757afffdb94e83ec0348610f54764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297281.13/warc/53f9d593baa2e628aef0ed5b674d266b0e8b2155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297416.52/warc/354a963c8e51891aab7416866cc4ce40d1ba9c48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297416.52/warc/403089d903143ca1804a2560f7e9d327196c2fb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297429.39/warc/a550e1bec1c474c314a411f5f4ca109a0615c33b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297429.39/warc/d9b98260b748301fae32988c482ac77c243fdaea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297505.18/warc/41beccf3e69ae89fa389864799600f25dddf3590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297505.18/warc/fea325a11eef235eee27a42a2ae7a44898836f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297587.67/warc/2488ed926ee0e552a992290eb3c4e0c56e8e3aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297587.67/warc/fa55a5721b36564de97a3aa98a0cd97f2716c819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297622.30/warc/390f68e6fce1868714c0567eb2f0af78cc3c1dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297622.30/warc/74ae77771dba8689b693e676030fd73cbdc235b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297628.55/warc/aa12527b5de4b8fecf6e9831ee5bc44f7e50926e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297628.55/warc/debbd452bd39facd1afea873d86080eda4fb131e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297689.58/warc/8dd825c4a55026dfe7663c26de040db1f68632e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297689.58/warc/f13463407e7b348b42a63c5c730c02f447d3843e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297691.25/warc/3376909166b920caf2ebefaf005aceb207912c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297691.25/warc/ad4508911e00319c609a67490ce03e394e9eb232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297831.4/warc/13aa4cd6dcc44c6a55b55817efad0e8953a44b11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297831.4/warc/715d42ba244d8aa609980044966b67a6c75c6972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298015.2/warc/5e6803aae88bf9f197e5366797ed8c2298c0f3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298015.2/warc/860a6272288d0dc8c0ca27705e2e7b44d93bfdb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298020.57/warc/60afa30a79cb27d50b56ebf7d13999260040dfb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298020.57/warc/d0690c17fc3f4024311519e49d60e025a98f49ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298080.28/warc/0e1d9ff93d6ae1dd8131275c5abbd2445b779880.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298080.28/warc/7656ba11711592730fc02a36c8dd2dd5d3911273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298177.21/warc/8f21be0efebad430b8a679db4d8b23b9dfaafd04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298177.21/warc/b60f11637dd0b9e5d626ad4e7d7cf9e4133b366a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298228.32/warc/16d1dfb5a4251adaabadca52846dda077d7ae9f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298228.32/warc/de9e7a07c9aa0127f8b99bebd901d91e274f206a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298387.35/warc/d0e8400b7162e8ee85c2c8d739042ca71c67a181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298387.35/warc/e1df724cee07bf1de9d6040b179d8340ac58d619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298424.66/warc/30ec38131fac063db3f49085eb826238880378fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298424.66/warc/512fd8c483694d3ac2c4924079c4a850dfdc71f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298464.94/warc/af7fef72bb48b36d74550d4617e36ba5004a628c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298529.91/warc/0fb294befc2c93300db2149d543525b2c4e801c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298529.91/warc/3d7b9734671ce3208a43066c975bd4c3106480bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298538.29/warc/24b1064e6b0d6ead4fbeca9fe9fb5999a1fb949b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298538.29/warc/76bdc2d94d2d6be928f3157f743d53e045f1b71b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298576.76/warc/a0a957e460751c58f365228eac9c732827019943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298576.76/warc/ae768a636fc65e0e49b92448d5d93a030aadeda7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298660.78/warc/495bd332647e9727336204663d9c49fb65f7cf08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298660.78/warc/a9094c01853a2752b40841bfcf80b5c023de9e7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298684.43/warc/7f2c72e492bce83be6dbea965299da39c9f8db41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298684.43/warc/f7a20fd64c7de8fb25fcdeeeee974bf183e5e16b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298692.48/warc/28bbf978e6cc7489c0ca3d9abf60206da37c05e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298692.48/warc/43980e6fcd4a08c331b7fbd33ab2c4801c74af35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298755.8/warc/0f8d8d41d15a660a77e6c602ac7756f0513e2f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298755.8/warc/26d8d5a8ba21304715ee3fd443401ba2005c2635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298777.65/warc/40d28526367f691fd799667d1ac0ec11ceb4dd85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298777.65/warc/9754e0ac80b7fc8cb0e7c22095e04ccc081c21f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298819.92/warc/29afb61162b67ee7af2d10f324ab62909c9221ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298819.92/warc/2e859d4085a2e69cb2008db17c82569e49e10b00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298871.15/warc/1b854e644e00e12cad74bf02287cb37262e2c7db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298871.15/warc/5d713fbeaf40e40d107be4905eb5166962be5afd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298889.70/warc/32762c856ad31b309658c1e8df99f787e7ab5573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298889.70/warc/e773ceae0d5c4c0408b9edb4bc6829140df7ace0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299054.80/warc/38a294f5620091592944471cf468be3e42abe972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299054.80/warc/583aaa5e0a478618f991a7d7f7408a2510db6447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299092.45/warc/c582e024e3a84958f2c5d87722e92d3fa4250685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299092.45/warc/cf1a25defcddff523d8478d5ce1561e9e830b7d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299114.73/warc/123e1016c4a4a2b1deb5674c9792b3c725c0a1c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299114.73/warc/89d734c57a06ece41b20d6c13311a2c0a27d051c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299121.41/warc/c9149ce2d56222e154ac83d0d90aa55a219e75f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299121.41/warc/ca79a98b798abf8a9b26d936f8fe90691b0277e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299236.74/warc/9688c1575caec77bccb7490a639a50912119d104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299236.74/warc/b8c41fdc3af7ca8690ad78f6d658c7a524514b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299261.59/warc/9d0efd25c2744defa22e39b5dfcfce6c9fb18c29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299261.59/warc/e51fc8dc8f96759b89a115d0f7b3b3ea50457b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299339.12/warc/4d770aba7fa714f0c0ab56256840066512e2ead9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299339.12/warc/88690d9bec6783b35cd86a42a4b16100eec3abcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299360.90/warc/1065d4a54931d845fdc69bbf472efa503c94a6ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299360.90/warc/13220322d198bd543d31cad2178409b575182f68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299496.98/warc/5d99774163c9910b6c552cd9afc065d4bf590abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299496.98/warc/d12e6981f4747575c2fec785b1d5064f7c90208a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299515.96/warc/4245a59ac19434d2fc3cdfac87be9ef49c2ff09c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299515.96/warc/cb9024ebb9334644ab925f4bbea4677d63e84143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299877.5/warc/b3f1a230c812d3b40c289ab4cf1c19dc8a021250.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299877.5/warc/c425870bd77b639d8b4bf7a7c22be1f66123bc61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300017.68/warc/84c5fb7efa9a5f319aed5361ad10f6f4e302e1cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300017.68/warc/dff86c19236c7c3a9d919d58d59accac63100af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300031.56/warc/6727384efbae40556ae2d773c0fe14537c7d96c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300031.56/warc/f2f352d25302077da79adaadd0a5081dabfb5b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300031.99/warc/f0864aefd6f81fe0d3e9d38436a8ce649b9202e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300222.27/warc/7bb4a59f5ce547c061994d29e320e6b21beedd44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300222.27/warc/e9e20e8e781e260c0a17214baf2ab70ad8c751ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300280.0/warc/24cc6150b1ef1d8fdeb27cd083f4208117e09fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300280.0/warc/9535f395677745ae2a68bc78636762c3343a8e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300313.14/warc/df3ef85a2e9eaec88b95b9038b92ea91a503649a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300313.14/warc/e08fe5678e7ccb1ae9609636910948accfd8b490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300441.51/warc/25173bebf84feb688d95433c1b212de49e830585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300441.51/warc/326054e0ffcc536e27a8a6df0907ceba4b3d54df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300444.84/warc/01aaf74a2a5868b3c941406ac788fb241b5ed3be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300444.84/warc/49fcd68cc1307d8942f7dc669eaf6636e66a6ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300464.72/warc/170611ef1b612ca856885e16bb02c1933cd61ee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300464.72/warc/74f7632bfa1f16c1a37eb0709483a3b16b8898c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300472.97/warc/0bd05a102a0067afacc757e7fd607ebbe08ba37c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300472.97/warc/5f8f448fa6e5526ac99f4a9098ad47f8183abdd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300578.50/warc/388639e8ed71d538dbc0e59180ed96bc4ebd1983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300578.50/warc/3bf09d954f6f1ea55ac6206ddc1e84ce60e30322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300735.71/warc/94fd97490ceebf7501d6113a14b552f66f53a706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300735.71/warc/ddb883319ad19b237d6b3383807b211558988b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300773.77/warc/0ec64f87139325609f65872bbe6b123a9407737d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300773.77/warc/8f679a32e8175f8e714eb33f45b56724ef4a04fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300799.9/warc/1b7c9b826eab54ac555fa3414ca5c4db576e5a77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300799.9/warc/98f5dcd1cb7e041c534b17d0620f7fa0a9c34c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300905.36/warc/1cfb10e9a0780259f349771539bc8b371f27c0c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300905.36/warc/d148171c98060b59cc66560581ca6430bc36d4b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131301015.31/warc/3f60def855b1d12a89dd8198279cf8f76d7aaff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131301015.31/warc/78da618cacafc29fce46ae39121a84b87287cc61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302318.44/warc/0fa3da9380039a039b676c3a4a22841cc82900c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302318.44/warc/7a4d7c2f228feed2a6d1b1ecf3026ba56a060a83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302318.88/warc/4671d571dc6940d299ed410fac029233cbd34697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302318.88/warc/a65a6cd8f8275308c4c05324b9f24fb81bcbed4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302428.75/warc/96ae23a4625985a0f005c6f9e5b9b625d48eba69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302428.75/warc/ad6cd3f31ea9e0ddc50bfb36c0ecd8b578c3c55e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302478.63/warc/b6d46bd8b8a8a868f8e2e87480fc55b71d31c8a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302478.63/warc/dcce5d8fc4e607694bca029ceec7f7b839770dff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303461.20/warc/1d19993ff4008c1ead9b275ee0d0357bbe3e4f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303461.20/warc/bf9cbd2b4820671bd465b30e08d91066ef643903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303466.87/warc/bdb4de1e56e9eb48f9707193a0b671cef7666c00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303466.87/warc/f5d6fed0495d8741c02e4af2b5e8bf12deaa2c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303502.37/warc/09367e455a4381cbfd96100a0acd5455b33a24bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303502.37/warc/2c36f68d0d59ff7e67212760361e2965ac79e1fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303523.19/warc/1207065b1c13484b40f9adf760c49dea519cb020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303523.19/warc/c7c95fe7961e0d3acffff19b6d3543586b10ae0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304412.34/warc/36314233ffb685759ce963abbd16a889d0717e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304412.34/warc/79e938e31b84040ae096494ef08eccc9923cefb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304444.86/warc/9c474dfc21fcb37e672d088cef0f912604e75432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304444.86/warc/f5018de745be30c980622054a7921e44268a2a33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304598.61/warc/22d4c5c64b34fa88e27f36fff93104b52da06782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304598.61/warc/f1eb1ca682662ac309688bc899bece78f7a9fc0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304625.62/warc/9b914a75b5836039804079f56f6f916d8396b60a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305143.64/warc/4eb7bacdfd9b89231338c400a6493f894c659e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305143.64/warc/7345ec00a4e137610ee6ae1d41d3ecd99b5a999e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305143.93/warc/438451befa83616b6e078409ee2c928d00c53a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305143.93/warc/7a5aa18f5ee22e79e42f3cf10c3c02c25f4b3741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305388.23/warc/20ab95763ac9de777372ac05c7275990ba7e2298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305388.23/warc/35fea9f79fd8e86daa6e996c60b995565d3076ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305484.69/warc/dc6607b5b9fe541d0c32aeccc2ab3db1a78134e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305484.69/warc/ee8fdd51d7247c4f2e7ff34f507e6df2da23823c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309963.95/warc/a02653d06ed2ff77fe2137d5902e523e52d560b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309963.95/warc/f59bac246b0bead5f4abaf6723551c0608b21bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309986.40/warc/1790fe0f243e6146d833507be0bebac2da916cfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309986.40/warc/26eb9431975ecb5c4f0dff1120c568d5c386a121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309986.49/warc/8f0c9f692278bef1322d0efc7d39bb409c1eb54a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309986.49/warc/fae99df4e69be967ccea763fed3df4fce9b57c4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131310006.38/warc/716f003dcbf2b718527738181702d553fce2f2a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131310006.38/warc/a5cf41d191612064f9a1d18ed8b6fd11465e8c57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131317541.81/warc/b8a6aa08afa5c9780b08874bb12b6a4cec071421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131317541.81/warc/f34ea77c0eac0b0c2f2fdd09d20ae644cd13fbbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131317570.85/warc/b39ce78005aa887cd73bf2ecf9e78a58e686bd96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131317570.85/warc/d12a9031c44749f0472a3666f6fbd359045a817d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427132518903.47/warc/26610699e1b7e0f0dab9ce42b1072011e507e0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427132518903.47/warc/99ed0cc361271e0509c0f94f63673fb040d6e8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427132827069.83/warc/24c4934b4b3f115b48caf2de8c11d1c4f5119416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427132827069.83/warc/9ecbfe79e52ce7aefe57f10fd6a77ac76f643641.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1c56da2dc6cf49c19c041095e572e0c34076e618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1f337d8b2b0ace678d584e4d4c9ddc77577395e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1f50d135d0d3229834f58d4090fb04c7383d2470.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/283608c77bb89603d143d63725913172c752efa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/2855dfec773cf237445ed46bafaaa196b287bfd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/2c3ae72efdcc7f4afc68e416461b005fc20d542e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/2fdeeef5c9466bb94736323ddc884001ebd8e456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/32e365935ac31aa8dbcec857df8e931bd848c286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/3d3dc34a79de1a62f6510e01c1b962cc9221c790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/407c7c3af7fc3a91166e59191ee6d709250a8318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/433e77bc6bd8e911136d01ece9565465dfcf1bea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/467cb62e28663a0af8b2af83eef0564a649aa60b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/507cbdee729a199eac6af459b77e4a2a422f6ab9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/520eb6425687b50df12da5d047265071cc149d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/52309613ba6a56b4545a309c3071df7511eb6522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/5f279d7f4ee6c027e427c01db38c72b91c0fd994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/62d3843b5ce714e041335fc2789695c1b45736eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/679dd784fd1613c42de5a607430738881b3f36fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/681acd9d58a88d7fdc15a69c5660de5b4d600bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/6b22003392a63627110a9e748790225ca80a0f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/6b8dca6ea9f50ae8601a6c2af7fc0761ff61a371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/71c11fdc3bec393868fa72bb55c3f1d758a58665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/71e3173039502aaaca61006cf16de6a2b042e811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/75550cb605d28b2f98c3493dc90d07e063b6c46e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/773c5075c5b8a457bcd8c1a1f0019e86295f8f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/784bb9626a76061ae554b35305dcc6ffb78c1822.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/797b07437601c0b2b0b7a08c4fcf769800b083b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/7d73615c4d0af72e55fd0d4218279a23a6f78000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/7e31a12267e75edf07346cdc153f8dda0d1c604a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/7f87553e42957e9310231035578db8b81f645f9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/805d91d6dbd64b3319c7a15c9cf696779feb83c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/82dc999418dea1477d2390860d2bd4385ad5dcb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/839bf027e320b60a1d08e030181286be5a15bc4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/852717d96238e6c8b82551c1ed67d04d974d6555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/8747b5ca0613725018c739850291e805ea4a941e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/87ee34d40f0e640e5828da031af796a0cc446801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/8abfddde17d121edc0200735e5de8ede16472e27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/8e3c42208b0e31ed2f75453e59c3d9f18ac9058f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/8f2e7bdbeb3ae1bba3aa2a9eea192cb9ee17d8d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/94f46f183766715a4d635da784deb65a74da179d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/95753dd5ab53fa0a972964e4ddb68b4eee8ccdbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/95fe5b72e594ca55bf788117d8ae8b7e3a314e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/960dbb090af618cbcdea43bfaad38a801f64108f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/973d6d8f1d852b87903e887bce8f2070bfdfb21b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/98b7565df5be2eba9df617a654b56c391f308adc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/9d1e9572d1f9ba3f5cb6908213fb197bb0de6c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/a1b575ba501ce5c03a1b13998f9dd806355fbb84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/a527e7b85e76ac3e42d2f29f956050b34a0fa2be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/a59439b90c1bd977246a65b764f4407a69ec5445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/abb54d6235179f10bce8c2178fda0f7345cfba08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/ae24fc3f338460088d9e79c18e3c5dbd52e3e650.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/b36039baf4f523a0fa886777ba9c374f84a85aed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/b3b65ae6adf269498f120dd18118afaf14d73de0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/b75dc60a02a6428038556aa2b1b3c34136e82560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/b9467af637f7c6a52da0335bb533d763ae18172a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/ba7227955cea66639064d480942bb7398b5295d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/bbe00948d7a747d8a19ded8bdb95d4ba9d327e00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/bce72c73eee2295c4f5665f916cc6b99ac158df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/bcf977f6a407d88b628be6bc7bc15f70bac095d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/c0e493eb71adcf0c0c13930e17d81b7aaa308c83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/c66eebcacc9221ad651324f8cfdcef9c77e2926c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/c864ee120106a9e43eddbb36815fe4968092efaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/cc0aa39dceb146d1dc186a275dcf76d3a6d9f932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/ce14f954cd0343002d02f4cfd56416e9e4f73f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/ce438305e2f4e88abb6efacc346abd6f2ccc5579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/d04c14fa5e26122d6816292429444e44e13848ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/d0f0658e6a40a0096932579442b7e9373d582226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/d2fe25e5f48762e1d6199759aeb9865f28fda08e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/d472f8bad25b26b76b95ae8be38c23db851ffdfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/db607c74412e584d3e8a45b7426afea2a04e3996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/e2bf5fd68541d52b7f1fd085808d3d4e71fe5fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/e5870943f84954778948cf87146df32d901d3a85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/eb6cda3b422dc5f975915321e1ec700b16318737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/ecac72942c623b4294f65e2ac8f5e5a5fcf47740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/edc0a03775a6de39af156ba5f667c3e3ed0e56a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f1439b491ad6320a1a2ac4cc1938a61a8f96e26b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f1824327f1008f19c80bbad8411cb592b23c6a06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f18c669a69267dfa74d97874e96f3f5300fcf206.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f3e488b1e0983544ac7160add57bb409255f83ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f5af9b81d69c0313540f2960fb4df94c6b3e9f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f6369632b7c04fa10a4053f1a8523abaa839fd1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f6840110cbd6df1bf1b8984f34e4d939083d3e94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f80cd8fb68af9d14023602402a138b06dc050ae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/fb43a23af497a502f5bbdcf776f335013b1332c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/fd1936254d98e588202abecdd4a3aa3d78730458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/fd93d40acb2475a50f3313a97de06551adf12cc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/fd96a741b34e4627683c8d0961e5f3b71df6b290.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2015-14/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2015-14/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2015-14/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2015-18.yaml b/configs/cccc/dedupe-url/CC-MAIN-2015-18.yaml new file mode 100644 index 00000000..6c8c00e8 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2015-18.yaml @@ -0,0 +1,408 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/0111139cf1b363ac4c4a7ad8c6ebfb65ca581af8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/02d7ec60b68e2c542f2da61ca2d40869b6e072cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/0324e49782e208cd45632d7261fa8f7f8c7cd8e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/035460fa1c4bfec9aeeaa2123cc286ab95fa4e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/054af6e715064323cfd4807a973a583199e34796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/05c5fa2f00e50e56904d5be18aec26ea9adf5c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/066a195bcc79b70d824bbee3125594cee6ff0f27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/08ce5504296ae1879fa08f659aed2beb2ae2a941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/0dac07a56582f25a68170e9d9378432b61b02379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/0e5beb50418201e653d8ab99d7a559411a2a6eb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/10aacca0ddae537d3ea2d40151bf65bef42dd88e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1372416cc9998668813200b4806e244ad6e4ab07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633512.41/warc/602cc887dcbd38af83155d50d928c62f110990c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633512.41/warc/e40e15d076589c4813d378e50c9c63eb3b66d814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633512.41/warc/fc120f1453747d8c8fa56d57b045f628a51a9394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633799.48/warc/5b80a560b5605df7f93023060cfcf68a46537db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633799.48/warc/f2f6cb08fa0273d1ee9bd07453d6761cdadcc8d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633972.52/warc/85bfd7d605bf0cb01710aa8b81e6fb496815ae2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633972.52/warc/a72fd57c86b5c5e3cd07e163b5344ebb44fe1d8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634257.45/warc/5b03f7384535fca6691a892d0ea5d006a8af3f49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634257.45/warc/aedcbddda0c1176f31cdb8f4c6cb9f17ccd5d953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634331.38/warc/19675a5721d1b33eda9f41dbb403b80db726808e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634331.38/warc/9e7d90f3fafa4b3574a753df3c0b4282f7b94671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634333.17/warc/216e2740a23f493dffbb524a987327ef79ce791b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634333.17/warc/36f3b5d8ce8f99725c95200d81757f13d8fe49d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246635547.24/warc/8c99bdffef88f0d760bf1f8623077bb970dc6535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246635547.24/warc/9fd28f9737d1b4c9d5ed108faf0cd538ea0e63b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246635639.11/warc/3021e5ca74413ae27351988a7fee5c95abddd86a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246635639.11/warc/306978e587024d40cb5ef6f206e42e7d6013d5e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246635639.11/warc/e3a7e676951fd90a2917980090e9a30a77623893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636104.0/warc/a1211d03cc029d01d6a56ec27c91cdf7a314018f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636104.0/warc/eba9000109522110dea9271679ab06b253e3d5db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636213.71/warc/148be045105692a99dd714d25c21a411fcd4e9ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636213.71/warc/98fe467123066f75d405faada5d53d81c7409c04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636255.43/warc/66e48ebc447d63deb5c6882491c3979b06dae521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636255.43/warc/f207df33ba2b0f584b10caa2720fb764de0bd603.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636650.1/warc/17fc4c3c5453a212e28ed53e643a4c3b5a160d8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636650.1/warc/ac91b59c86318c5f0f8c40304aaf3923c0e79b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637364.20/warc/108599585b1e377a56cfbf65587a2e4783c4584b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637364.20/warc/273e110ac1df3f4f77f4714f54a0b8272c8f0e0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637445.19/warc/50a6f3c5eb69a8e24836948aed55b4557d1df6b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637445.19/warc/89a296896e40a9e4f6e07249551976a59c73e9d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637544.68/warc/47381300498a3ce436bf1290bf98a60b393ca2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637544.68/warc/5aaa6d5ed2455a6598789e4722ce99b0aff1e17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637979.29/warc/11abbdd9b1da37a3539b8e3322e5fe5eb5f8669b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637979.29/warc/23743a036e1b5f98ad8d3283b7fe4da2119e86fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637979.29/warc/445f5e8df42192d8333e6ed3ab4532394089e9ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246638571.67/warc/1f35973f07403131c0b043b6c381f8835d59948e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246638571.67/warc/651fd5006053d96258e33678abbed2675294afe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246638820.85/warc/52e63d1352718afad388af6acc6b951bc015ac6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246638820.85/warc/857cd79b4e2bdf6ebf14d8e41010572c8a95750c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639057.4/warc/e0b92d33799dff8ff37bb5f07aff497e841b5fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639057.4/warc/f6a8f4e3c314267f1cbcd1d359a61ca17582ed5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639121.73/warc/1eff24c453c818301856bfb28c1f8465734b1943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639121.73/warc/b53407e0809f3387223930726902d798f3ebfd1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639191.8/warc/4b676da15f7a52b69182c4d456d051124d73f108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639191.8/warc/5a3d4ce80ca6a1d3450e187de2775b99a8179d3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639325.91/warc/7c615d50a5e6eaf3f6cb9899d332cdff04d9efde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639325.91/warc/a5d9e4b2a27b77a14546a73af531f44da2c99f99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639414.6/warc/22a4e79c590f4a32dd6f3f5d28c6e754135690dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639414.6/warc/e2a78024f777d0803e11c40f4aa36814d7023b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639482.79/warc/3d2bb8355345e572acaf118092fca3a9adee53cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639482.79/warc/b2a2c155a25dc1bd4ae8653d24d2d0d4ff10ae8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639482.79/warc/fd51bcbe440321b03be88311961eeffc9f4fe5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639674.12/warc/0a1ea0ee36ebce2614896475e4566f8c2f6baf01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639674.12/warc/935d243afaea6235699ad7a8c6e224257066e883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640001.64/warc/2372a1c8335f380eaf0a7e5894bde4d5835e95b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640001.64/warc/6b798b03114081eb7838ced1ffd899bdc3ae51ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640124.22/warc/874f8ba9351ea69218c088d94d273940698a7889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640124.22/warc/b39ba37545b1fe6fa6a16bf94fc3ef4454eacab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640550.27/warc/8ea32c28b00b9ef56aa9a95410c4576b98b38d2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640550.27/warc/edcd659229345158fc1940f4752e6463e84960bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641054.14/warc/18ef8ca82177c3d280bcf193ae60ea4cdf9af308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641054.14/warc/403a12bbec8757a293911ab873fc51c88784a870.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641266.56/warc/208afecce64b3e7177d539896e1c975671763bdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641266.56/warc/49c1715dc86ddf4377781838de26655ab12fdb2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641393.36/warc/15fe16f8acc77de61f0a05aa5650bf9dc84795e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641393.36/warc/e0037d8bb42effb301a2e6aacefebaa9da9884f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641393.36/warc/fd5570c93ff874f06891e2375b4a271dec66eb7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641468.77/warc/40986e755483ee65944b054a0501ef9f63977f31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641468.77/warc/911d7f69de81087a248e854047e96a5db5d09eb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642012.28/warc/30f00640759bdb546658e99c1dd8deb6e94f03c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642012.28/warc/a12fcb970243914f48071b0abefc13df4a723f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642037.57/warc/25708fd6dfcb965641fcfb0ef4e0cfc65ccce6f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642037.57/warc/7bb5d21069f95bd6dde09875b604742b572af1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642783.89/warc/a87f8add4f129ef31148f485f9c530603b0173f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642783.89/warc/f95258a30ba01f2934ba6424766a3bc421f1637b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643088.10/warc/8eac5b1e1365c648b8a65c1f818cd17d68852f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643088.10/warc/ed0af9862ed044eefd0e1ad4731bba5a254aadf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643283.35/warc/212c0d56531f97bf7f85f9fae0948add281021f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643283.35/warc/469622edec6dc98e7af2c039ac5bf2c58bf516c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643815.32/warc/3c22b85300074e8ea293d33a3772b21fcbe4d3f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643815.32/warc/5fafc0e64313778542acdebb7bb13e0c0f7d725d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644083.49/warc/2b5f3663a4bd6e2a7fdfcdae5a2cc0d867e1057c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644083.49/warc/dfa619c031ee2fa3b091b73c626351c86d88bfe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644083.49/warc/ebd1ac2746c3821937d0f028fd94284a9cee2c2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644200.21/warc/3d6b26cbd60cc66beefe4061294b2e0e3e94b48a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644200.21/warc/486696d342bd1420f95633c10562e1fd2c6552ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644526.54/warc/d2e132314654a9fbafe3e7638e81c8f65d24bf89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644526.54/warc/e0e891bb31fd03e2d9a968e8885e348410b5f493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644715.25/warc/8bf52836464754069e60457797b1474a7fe426ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644715.25/warc/9a2e9d1cb9d1285da6a970cfed7bc2b1c93e6a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246645538.5/warc/3a722531adaa75125ffef0faff838023d5b245c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246645538.5/warc/ebc16decf4faa2ddc2308fee62fc144f114f04d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246645606.86/warc/4f27a80b13ebb47085fdd782de0bf7a5c32e7ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246645606.86/warc/6ae71bdce797331454d743114f8885739b217fe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246646036.55/warc/3a2dc45fb109a2ca62be98bc7098a103041b195b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246646036.55/warc/fa5674c77548880a75a2f96707abdece5bbebc63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246647589.15/warc/0394864941f8dedf1a12668a1ed28f62070f485a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246647589.15/warc/60613427b7158cebd9a00b03c97682da0e8297e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246648209.18/warc/17bb4f07cb874db13f9cfca66db1f4cbf5083d4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246648209.18/warc/43fc163d566ee6377ac67aa1c98426d3d22b1fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246648209.18/warc/a1e897497967f4e6fa01b4fc0c70e58b3fe7ea6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246648338.72/warc/8c82999372c5f35c9c48efe798642a87da203a45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246648338.72/warc/97ff64dbdf5638676a19870a42cd989504d07311.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246649234.62/warc/696d5ce699f974152ebe38511ad6e54808db87ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246649234.62/warc/f248babb5d5f7294d8c4935f053f9f4a4ab89db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246649738.26/warc/e92deadcdd8f5e17744b44f19bbefcc44880b095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246649738.26/warc/ee01b49e348804bdb891f554d15e0fa1461b48a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246650195.9/warc/d01b8ddcedc17db57f18090bd1986339d491cc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246650195.9/warc/e2ca8108b28ccbac89a60c07d019206cab0ba7ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246650671.76/warc/3b8796c5d823f7837426eac4e11ca968454c2645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246650671.76/warc/9bb21b24c2c4a293b23d3dcfec2e6f8646c7def2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651471.95/warc/2dbbd8116ec62b8cb69fa714a9eef820a4b124dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651471.95/warc/5cafc5f429b82e5e55fb16353f44624123cae008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651727.46/warc/0a81c99bcaef57ead592e0f82ce4fcb6cd6fb54f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651727.46/warc/d9473ec79f41aad36f67ad8f84d891d5d9e1f619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651727.46/warc/e2c619e07b35c668d1c8bd50fbe00f13cfa1277e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651873.94/warc/38e6662648a9429ca45e201f4cd75da944c62a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651873.94/warc/c41043ff60a4ec6cb587e751e6be6123d79c2d18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652114.13/warc/4c0b6a49d1924ace05284d5d9e1bb97542d9194e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652114.13/warc/7635dbc33c4846ab5659735e11d19258ca1cb536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652296.40/warc/34accb451a0388159b23b12cc2a96511775f7b43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652296.40/warc/5d888aa7b9d928adecf201085856ea24e1107c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652631.96/warc/2b1b9d6b11370f8162c8b46b293558c24c01cd38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652631.96/warc/9b88bd195330931111ba9a4daa0a04d175086795.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246653426.7/warc/b26755b13dc8e040e19150f3477321df141ce96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246653426.7/warc/d2be67b43749e5c645fc64df8b0d8b4fba450b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654114.44/warc/af2999bb7e91a8fec0b2e6cfffeda53336a75896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654114.44/warc/d0e8c1705f29ed79e04606c9c2afd0afcc254d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654264.98/warc/7ee23f5d77abcbe769fbc3c051f41a42ebeb9329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654264.98/warc/d59927115adaacfdb78afa5db8eb503c845f72dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654292.99/warc/182ed59bd88b05f60f82063c86888a1675b03f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654292.99/warc/3680483e5a74e6e9abc4b33ab700792b03992b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654292.99/warc/891777525de79bb4a66199613c6f78bc2764ede1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654467.42/warc/09bee76a8edefd055aa0595d8d50385433ee07d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654467.42/warc/31650c45ceb3456b7a97b8567c072056dcbbd634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654687.23/warc/43a40d8bb182e2d3ae4cddcc5cd820fd55a20b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654687.23/warc/8efecce6058db2444cec09525bbb3bf9d8feb9da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246655589.82/warc/0f937bf4cec654394a38fee0ca49bec80099359c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246655589.82/warc/ac8aa2a7498a69ba97f253bec215d72e6f022289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246655962.81/warc/1dd2a24200f55edeadac353e066ec0c484f1e943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246655962.81/warc/62a10ec1d757726343567c3b725616401e5fa47f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656168.61/warc/17f363a73fc8f8c0191ad034a14bf831ca46be26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656168.61/warc/f03d725ee833cd4e4ff7e8417a86a1d355d8fec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656747.97/warc/3a43177fa51f99bfe26731699421f8cd9b54ddd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656747.97/warc/fa5c58fc6718c78ce544386efe89bbbaa5a2e0b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656887.93/warc/0dccce2009db00ae8c71ad42c9d37d935cbd4695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656887.93/warc/3fc7198f0fd2af8cf1459dbc72587cd3eab78375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656965.63/warc/436158bfe283a7734330d3420ea5c8a93d1c65ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656965.63/warc/69de430df4452fd69f2590bb96179fbeedb47015.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656965.63/warc/7d6815034d840c439a6f7bf268bda015d58d9313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657041.90/warc/6ee3537b74725a51787e601c876e83dd7bed2536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657041.90/warc/9ecf69a53ebd570ef76f7af0d6a63e23c56cef7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657216.31/warc/7ad69b8be17984381eca67f714527804bfe59c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657216.31/warc/824d85bcfa41f4f783dee60432169a3ee25b2d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657588.53/warc/d059b0414c77e9bdffd42a017f6d32f67cf6189a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657588.53/warc/df3cf38d9f9d53cb08d36b0504300ecdcf27712e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657868.2/warc/65a10a3f8277b41765c48ab8f261e4f7c966dd9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657868.2/warc/b2000e521a554a2348902ca543d7dfe79fc05eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658061.59/warc/ad0a6c5f5dfd85fed2a9d3badb427f5658c1de84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658061.59/warc/e921edc017cb7a22056f69eee99f1f06451724f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658116.80/warc/b165309ce97a9fd9c0f06f392ba932e83020b75a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658116.80/warc/de14174ad6cd0505fff763fe67dfa325dd5e3e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658376.88/warc/60fcf67870d122ef4a1d384b58d4883c789d45a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658376.88/warc/878e7902d8614efa827a73f4d6d8bb8d202005d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658376.88/warc/d73350d8d9ad5ddc72737900a6a7f7f3079ee4c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658904.34/warc/6c656f4def4b1778b3e24097a528809c57cf1898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658904.34/warc/d03f5fa19907a46751c3c5deccde3fb7ed540b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659254.83/warc/1c67aa8bbc199e0d0fb276801be0abffb5b5459c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659254.83/warc/3b95a7363a313d4d247b4c8e71b08a8cc6eab69d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659319.74/warc/014514eeae6a4f67301e4706705ba24cb9940d38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659319.74/warc/98ae95a77b099f0874cfc459a518928685996204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659449.65/warc/828961f56a35f7ce87b71edb4c6208b57e82869c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659449.65/warc/f38d021a4ccb6df90ef331df8dcfa4679371c0b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659483.51/warc/9e3f50aff93cbffb7c3165c31974a987f9185cde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659483.51/warc/a8abd92acc2b128d9dc57e97588be79434d80b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660448.47/warc/7d0a0f85b3622e9ba1b66fa07902fd54abc57963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660448.47/warc/d3f685fec3a16b1055b9992fc92c92973e9d5a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660493.3/warc/a281f257452b9928fb83faae539c8def6809948f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660493.3/warc/febad806fac6a84a6391560a31190580c21d438d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660628.16/warc/0e47a594620d5787f5db03ede27a5b702e7867c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660628.16/warc/5baff12f20ed68af505bd04d12ce39dc95127bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660628.16/warc/e48cf7f68178e4bb6dbd1355ec411a3a743fe510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660724.78/warc/0b51211096904fe39a75d1052ffdd7aaaf34e470.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660724.78/warc/675277aaa0d10b2b2639a37cd0b415ab8cc34ccf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660743.58/warc/1f0f259f8abce75f7f46abfc73c8dae45d00d99d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660743.58/warc/cbbe1d1096f27ea0c394bf28dee0f8783f55e370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661095.66/warc/b83baf9cb83f7e5a4821e09cfbbef4f4b87d6821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661095.66/warc/bf1e1ab35490301cffcbb6b43294d051747c5e54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661364.60/warc/34b822f49d2954dcf18e53db24e42e22f1671ed2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661364.60/warc/9d3f481d9e12bc1ddff35bb1f7d0ad463b6feb51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661675.84/warc/87ab3bc59d64bd24b29877cb50a198044c549ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661675.84/warc/9ea6ea771f22a91ae1af0626489e17eb632693a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661733.69/warc/1a21b3fc9316d0c40f1a307edc91c64fe225302f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661733.69/warc/e771971e9635f25e9369817f2341931ae70924be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661916.33/warc/3a3b884cfedfdb3f28a2ce624b17b909bdf9b93f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661916.33/warc/fded9a96fa072ae5631282a403c46eaef0d1d7a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246662032.70/warc/42ee0c6871ef0170cc46c5781fe088a16a434292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246662032.70/warc/a3a134c80cf5ae8c1e39b6aaa3c336d05331c396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246662032.70/warc/fa1e716b3fb5fedc47501875bf6e1462174422f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1430461119624.60/warc/e21679323df7f99847a757e9035c5ed9ebed27dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/14f096ef914c0725db2ab4bcafd8278ccc05bfea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1693ab8bb1b5c1d855c2046033ae55bcab073adb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/18040c95a01a3f2f6cd3d6aa14e1f5e7fbc642f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/18e01b70706dd0ba10b9d45020e325a12936b098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/198420c347a97b353343369abc48f4ba8ab185b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1a118fd980588144867ef555a63d80a6b22842e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1a5e93f535faf1da61a95ba16d3ef0c672cce4c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1b0bca436c6edf9b592dc86b173094307b66e855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1b60d26466bbf9ee18f02fadbe747d3acb800894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1c5f70a73296ff40c49806b5dc2403fbd6272e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1da3602ca59438a020bca3562add98bc4ca43b12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1dc2cee3884048841372845b8162b4ac2a93a7c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1de5ddf8bb5d43473c7a9adfac387aa43dd24335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1e05a0af05517cd1dc7598529c10a677ce434907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1fdc9d549a885f6201e214d5956c409fe69550e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/22931177179dcaa926bc61b517bbe2192b87aa0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/24379c4f5e619bf82c682292d93a7deb814f3048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2522da0272651786f112e9caa85c094ea74789bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2674df9cc6a60b865edf94a5294e982dd44f529a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/26c5bc9ed3f9b1a4af2c553c2677803fd120de9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/28e78c7e6846408afd8989de61aa86459fb1c591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2a828c5d2a2a41c099ae444f39a5f03a6987ac9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2b57f253960521f2bf45802ef15d617cb1695aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2cd699c33fd444c392c64d0ffaa626e15f59b1a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2efe10a21d66d5ff90435fe1c89281ba391bff15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/30ea57743a3d859667a7a2f1823c713670a823d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/315d10badc4318542f2c97ee1bac0cd205bf1ec6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/3184ffa55608d934a351e3ab0e8fee8463edc566.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/343d17229ef8fd91cf0f9dc2c13b1a251e4224c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/3463c5ed1ed38a1dc8cdf5f6f3b112ec843db254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/347a3b3782913a4c5d24726f84263bf68a2428cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/350f7cd2fa2b6d791b374305c73cb6478e592a45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/35254f0abc616b0553e908442001e68e0e8a350a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/35df306b8ba939ce54886e8468e0d87a9b153761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/35e91b1f3df55d2f15e353451a00e1fedb36b2d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/36726fe106a692ffd417edf8fa65d34193e5f4e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/37c5852c90d796af70c078e1c3980bca631a695e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/3dc96ed37465634f100c4b6cd09b7c8430a0f77a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/40ccd5fe8a692708e4546a6c070a3c6f477dea01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/42bf78e5f67adcd1755329fd5c2ff003c5222e7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/43cab8bb17997f4c3bf62fe13a006e2a6acfdc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/44c6c0521bb08d836fa44438b9a87f37f627f318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4562e4795e4b8e3521ec8d3334c87ff0aca04454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/46b2da9b67bc6f10bdb52ea11fa200a8abe39e1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4b476bac29bf02e4c281a3b2a74e30a1b2637275.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4cfc478f23a7b14b2dc35da5f49190f199f030d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4d128c893329b8c9b7c8035dae39662143509be6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4d387e30203089866d6c487bfd1881570c3c6452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4d84304f69cad794d55b0888bd1a28c2be55ac21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4eeaede0bc5fa4f96f71a8a53722c0a04072656c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4f91cfec7e20776f7df68ffbd1e76a5652ad2218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5362eaf898e020a80d248ee9e60669749fc9f98f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/536d2672f7a052ce47632427cfc94293b795edcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/573ce41a1c7d0a7d5bd85a6ab68607d850886388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/57ffc731e94693a507990592f499251d0d029f42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5aa54ff2c5f4b4c306650d087a0c90243f64c245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5b8c6627fdaf24cb9ffe93d3ba2c916854910be8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5c60e150142df1df72e32d6992335554d2d85693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5d6670d92c01422a1ce722fbb9b963a7abe6a047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5f9aab553c31d0099362841320056a701c19ea0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/65a4b32fce14a2f1b7229de41f6f239a1308edd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/677ce27b79ce12069e2de81a606b0656e1fd0594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/67de3ff6f5dd75832b25dad3e2cd578491575f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/682c8e6d306626c8d6df9c3cd28480295116ea02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6840d30904c03e18308ec5b00dd8f11f39367489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/69acfcf99ec173981c3a949f61e4c82e720c331c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6afb10b3ddb2255dd7351840328fd03f2c1f07c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6b33d89d0b017b73e5537deec1eabb572f012ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6c4f6f21ebfb5039c750eaaada877c213659bc26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6d97d99c0ae7b13cfb255b2ade41a82b9408a7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6da3b59a2ad2142c166e368a389c310803909668.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/702944fa3605aceae1793ee170817be146f6d301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/71949966a49431fb77a27272156b669891969ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/76cc0aea7b72ed8769d76295c232b2d736ea1078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/7765bde9ecf4d99d38a456cbd1688af39d628d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/77be92c001dbdd62580867bd61b7a0d73358f471.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/7a508bda8762e7eb5dfaf9046b18a507fb984c0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/7d9964015c9c4f9bf97fcb2d426214b98eabf327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/7ddbefdbe5fff676f8da75bf88b90d87de7fbfb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/7f307b5221afeba252fb866f89503610072d929d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/8188a1e3f32973a6e121ebbb8b2b1b452cec5edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/83b11262e71dd6274ab6ef83df6af944e8f0a040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/83e5add317abcab9aa9a32134d40710f2524418b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/841aa71e129cb0bb1d043586208f8a8af00b8cac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/86bbede6b0955af37ddd73bbb8862079bba8128c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/8bb18f697dc2b04ccd4c13ac83779193d5f884fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/8d8211df5744234012b7fce0c12b79a68d571930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/8f96b7cb246b56a6fadcac7a6718c24feee4098a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9065556e689d88979782d4ae3a70d5c101a47790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/92eca26eaa06c410e3445940aecb867d0352409e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9538fae833481f611d32cd4b5795193cca1adbd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/95ae7a9b3a9b3c85c6740a21aaefa93a572c3bdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/96043ded666c71e4f4c982ab358fe1a6c7b9e181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9a0f74bb227715853cece36fedce151e53f4f551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9b6e30cf225f7658471ec38926cb270c8a934741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9ce190cd4e349c31781597bbc1bc958b898c60f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9e777147fe9f557319e78d85389320f67bf1295e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a232d79cf3e855439eddaafefff977532124355b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a3ddd82c485acb6f1561ba50dbf5dcc67d90f931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a43833610982a3b460f1af90e7e24d2d6349c907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a61e165961856617a8036c3af790e2c620152345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a62f7546c0971e3d375c65b73a00514abc5a4110.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a7382207bd2c5a6d8dec4e3031bbb9187f86e8ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a7b79ef4d240f8fbc96458e3bb909dd2173f06d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a89ad4c39ae7edfba3fb1cac8094c1830fc1a9f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/aa206e4d2b9d128536d8eb7adee5ebdebc69a03f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/aad8392008fc6f5fccf053deacaacc8009155f4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ab458a6d241c73bba03329296e19b8229f75f545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ab8d972ed23e57d56136e57f689fb22d6a252cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b1361aa6d1050bc6a08d997e702f475f6e2a8656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b20617526fc24d3814d125ac70e4cda089266cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b282628e84185f536927d6224a346313e785893f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b3e0d209f8dc4fb36b255d57fc94b0d0c9a8a717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b61545462d3a30c230731af7eeee5cc833c5bc85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b95ed55dfb8170e83ca071eadc5cc3c4236149c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ba0951d53455c608fe932f39e307763d138f9b87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ba9dc3a1d7fcfd29397d3656832f91e3dcbf807a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bb410083cdd7c7e770db32380c16e39ec18f3d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bc6948acb9f49a0ef0b8a1582e123cb83d238c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bceb043d2a41e1fb71ba4aaf9e6021eacf13dfbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bd74e67ba97ee3a51878acdd16344049ed9cc37a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/be09ec2b5899834a48ff0ec078004af4f1c6da3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/befa4ee241dff122177e1c256b2fe79d391c1f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bf6308f77dba8c9ab8f490059e8e5398501f5374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bf6f82042d3d29790e8b9f923c8c5dc273ab019a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/c5a2dcafb4201acf079fb5680302b542926deb70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/c5fd16dab86cc221cc90c377adce78351e2c6f4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/c77837229950ea297c165558d9b13508694ca93a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/c8caff4a14499a4ccdebb6583f49cb27f6fd1b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/cb04a396003eaf0a5938670eee97771c1fe0ee72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/cc54cc1f37005b4ce29aafa88c52b1a88a15fb28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ccb75664a859a4dc22a2cd75b732f5606c003248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/d057dc9c17fcc01f531385baf3d4ed127d4db6a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/d25226d40eb3af9360696fbeafce78df47fccb46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/d615d65d9296ef9573186491eab2b45f52f32a39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/d6ddcf866a9b1b974e2d071c1a148f3d1ac73b41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/db5d99933e22fddf42c5fcef2d2f49f65a1edf71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dc88daa82c20f7017149c3cf90662326fb25a327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dca676a0c91c2cbd388f6d23e013d546d7084f8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dd8a1f1b317d5da645bcc5c54b6c4efa6e070626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dda8cb5cb68cd7cb35b33b863283520e37faa2ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dfb99463e3f06060671f962865e1732150f326ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dfbcc5e6fe2295b797f1e5186708e88d1318898b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e0fc58ec93c9379c3483606761500a63c9b14728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e2597d142bc5d49b419565357307a95ba146f6af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e3ee749cd3167c6f25710491a346c5e6c43bd794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e46066d06a7ab5cf6ab3cc7656aa8540a32b9700.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e4d5d110966f9edaba7e61dccf46a412f78e1754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e8e6080f3ae5a722175bc707351fde5861e0fac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e96b7a1de0fd4dc28da9276f9df14faf275119d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e9a60ce8963303ae7b53f706884b5ad80585d2ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/eab99084612f79e804206524a4e25e291ad23d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ebc45556882a4c4b1ad9a019a3c313802a18ee1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ec53a362bf59ddbbeaa719faad872ca4e16fdffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ed1986a4f111da8670e894856c24c547eb177546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ed9fa9a1e8d62ee413a56dc220af67fe130bd964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/eff9805d8dc94594db1feda3004d2250862638b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f0d4ebdb3101ac38c08b9805af731b3e651a9b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f4414060c8101d765fe4d72c6e2ebae6903d37b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f49ef2e882aef51317dfb7947e4382691d6c545f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f541a759c13fba6e18dc2c795745cf1ff1560d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f5bf2ed9b05882023c3a343b2c2cdefeef027f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f5cb9d81fc5f670c92f4be6713ac46c5fd8c88c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f66333601e2321f4dac14224af140e5f59a68c65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f696e80968fa04aea7a68681d31b1b0e51225380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f74201b3ee902d7d580a1c26ca8edd7064404f7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f8100ee37c92a7dbd7ff1e05badefb928394ee57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f8ca48d850a215ec2fee4b548681049640e82901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/fac47785f78300e04c8f2e9ee8ff1f9a108b5315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/fb6945d5abd5e4669b818600becfc21036af6dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/fc1477a048e4871c296f3318b7aa694a04cc30db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/fdece4eb03f87e48ff1c3a6a00d3ae776b4bbc23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ff27b6b5f90a2b95fc92005b4b94304110ff3a22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ffc83ee3cd130eabba4786a249d3c5019a0358f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ffff74a16284a7e741e5b106dbdf0fabddf6a82a.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2015-18/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2015-18/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2015-18/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2015-22.yaml b/configs/cccc/dedupe-url/CC-MAIN-2015-22.yaml new file mode 100644 index 00000000..2d2de35a --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2015-22.yaml @@ -0,0 +1,393 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/001e68dee3c1733ef2f9ae038443a07cfdd6680f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/00464b7e0f2b8c7460fd2424384ce60fc56246e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/052b462b9cf88b07c0385024068dc65c473da81f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/05f9f8183f3980137152a52ae96c8acf0c4ad131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0733cf088b6b33e2f708da765d927c85c054c335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/08314ba4f6c3c392bca8b2489915a5d558f22ca4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0906ac9975a57d7b88920f66037ca152c900fb9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0b5112b40747f138d33f90e8fba4ced120b11a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0b80884ed7ff91adf3612b7dae6dc620681a8736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0c32a88a9b5fb52a1e644b7902964d4bf496e525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0d5decdba0031e768472ef36315e71526f4edeed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0e1fde38c6ce92892025d68e027cd8f088f454f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0f70f7d2f4eda78de87e06d9ab03a6f49001aeee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0fcc1257767bd68382d662fc60bea2d3a178ccca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/12930032415a175c4830493e5a86653c8843aa70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924799.9/warc/4cbdae9695c5d16fcfa18086e665de7b694e0042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924799.9/warc/98de3ccc051270062df7f00bf62f408e43176de2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924799.9/warc/fae97f47f2b90fda8bc13014e636882d71cac437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924919.42/warc/e2e494ca5621bd712f0d85d6745d6312067e12ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924919.42/warc/f9e1c1047e23aa59dffc3110f5cb299e2d8d0e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924991.22/warc/0d1df86d09b864352eb3eaeafb5f7cc1ccba22b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924991.22/warc/2a1b1d3a50aa7a2e847f6f896d3fc98855934742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924991.22/warc/5f32c4e3e988cb2d54f85872feb536d804434ec5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925201.39/warc/1fb6917a81eea050fe788b9d1eec700ed4f1a78b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925201.39/warc/93089706a0e2a95b57e29d95daac6f476b2103f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925274.34/warc/18d8044a92706d7fef27c79d524b79b0dbc2c145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925274.34/warc/3ed84ff5eaac4c0365944ee614e21ad1019c3c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925274.34/warc/c68052c2ac5af7ba4ae0e9a199f2b29069b5f81a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925696.30/warc/450e6e3b84631c3988ef7fb4ba2f51b89488ed7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925696.30/warc/8c0940b16007d00c9bca9107aca8a47ba184f508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925917.18/warc/11e1d645815715b4b4a96a81b1924056ad21d79d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925917.18/warc/c786a46db58c76d18fce17cd2e9bdaca16089737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925917.18/warc/cd8595ef94e12460ab0e175832cbc546f440a7a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926620.50/warc/3372c531111acb6a0b1aa9e4a8081796de554632.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926620.50/warc/e1ababe31549f7fa052850ee65ac5a17aa0c8507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926736.56/warc/767ab57423c180dd2e3f16e0bea708a91b93ab8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926736.56/warc/909721d9497f7b616ba79df48093cf8e2f4c365d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926736.56/warc/da76f9be5b76edbdea60adaa37c7ca0d948e3827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926828.58/warc/1782811d01af211c90fb75be29c1cd4afac6ebc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926828.58/warc/83b049fcb51335e5b890d2b6c655ebb71c7eeac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926924.76/warc/34ead19fea0ef79f45b54a6282f7f15c0106556d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926924.76/warc/f8b834cf6d8f9a95c9985ba45cf0bc4ce1d87ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926964.7/warc/650d9a957a6c25b7eef3b7513b5fe0a9cc7fbbb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926964.7/warc/87bd46040e14fc83c16e93abfd8992e6dbb2aacf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926964.7/warc/9b97cc0bbc91ed88d8b8c54f30a7b6221681c760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927104.48/warc/65c4e06a1c31bc3407f951f44abd40b426b042e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927104.48/warc/7c0a1be87b47d590c40b6d20cd2e82dbae74a479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927185.70/warc/390c8e11af30521637764db40279b9cb51aae55c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927185.70/warc/a4918918ef7cd1b0e89d25f3cf4acf64ec614401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927185.70/warc/d53ebcac5dc1dec3d7969bfeaa8045dfe3d4c91d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927245.60/warc/64139f7bac606db73dd0f127c6ffe4621795a825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927245.60/warc/b023bead1a109f45aec61540e8100f52652b0b7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927388.78/warc/4ccba2ae8f8ffa8a0b5714342875349758c47175.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927388.78/warc/7d221a598231e9db8e897a2d25aba68fd10670a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927388.78/warc/a474e00a9ebc87783e13de0a58fce72fdce9f005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927427.57/warc/01899a4bc8d9693d1bd076665bebb946fa643270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927427.57/warc/d37a9bf04605ac33edc4c6febfb36e4b5b4766d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927458.37/warc/534cdbdf40271172e54091d40c7fda1d459c8fcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927458.37/warc/dd7d51088c0d3352c6575f36d747fc403699c0fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927458.37/warc/e3c6a92848d823a0073a70c997c881f71fb919e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927592.52/warc/10da5370230cb1585bd8d621f557834a44a7b734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927592.52/warc/6d373e33abae0c921fa012f41ee8fd15a20542f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927634.1/warc/3cf30d08e71a0863d67ed19c3ddc5b1c602929ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927634.1/warc/6416edb8a2471b5a666668ff8da7f617469bd44a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927634.1/warc/ddc8ae19df2bba1857d48c64e32092a39f31e64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927767.46/warc/30440d27789b13026a432983b26d199751f4c172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927767.46/warc/9fa112aee808f0c49072aed63163032dc1b685f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927767.46/warc/b3275758c8d6cf572511c7a41870623037cef6c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927824.26/warc/ae3731f8d1e3848f9ce9a901f188c2acd5129e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927824.26/warc/e1abfff5abf94a7f4b2fd0e42696aff1f9cfdfdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927824.81/warc/8706e7827b69d27c6ff0e11ae28a034672abc21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927824.81/warc/8c42797f9372f2e875cda1a8a21c3d0aa98e176b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927824.81/warc/9dbda9586a6d7182083bb3b4ba618c67cdce7b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927843.59/warc/275748ddbe9a5ed189f7e163b3376f964ecb41ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927843.59/warc/33977815725abfa92d50b97dcfcd39b6674bc4d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927844.14/warc/33723b1335a7c60cef799c27b173eb6f323fc931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927844.14/warc/72631a72577631e9d85649bc145845d89af307a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927844.14/warc/f1670b2561d3c3bb06fb51cc50c8df3063a933c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927863.72/warc/18b1490bfd5385298280f6e2fd888e3af05291a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927863.72/warc/760d2bf2f18ed38984100736aa1e773b254eaeae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928015.28/warc/22b03a2707e75091599da92ffbd8fe586dcd7db8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928015.28/warc/aa20f1359640b208d5fa18d67d644958835e8da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928015.28/warc/fa4832c0d956b415be854dc7c15dd38ec3093245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928019.31/warc/28c2044c236ea9d0749dd0c4e6955201fc84eb7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928019.31/warc/f1a6e103b4bb784236eab8352d8199924679c7e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928019.82/warc/51506c7410147bf53d5fb1d52330b5dce0c169f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928019.82/warc/67df325c1b90d22ff6dc932c516c1a5557e23aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928019.82/warc/e7233cbf11477a5d50892abc005b73b1d5415c45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928030.83/warc/78d36801149e64f5b492a8f70c4faa7155e4de3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928030.83/warc/7f0cd74697f7bda76a91775a914000389527d0a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928076.40/warc/15810c1f723f068c797aed60813121dd8f1fe4ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928076.40/warc/f657fed2d5319f0f747c91b4eef6668825f38b0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928078.25/warc/086017aa4d7d62f3c85750671d21019de713afe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928078.25/warc/0ba37bd50a50e439b410751893e9114d9fc8ea06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928078.25/warc/d10a984d87d398d5e1020c1cc01ce67427bdf46c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928102.74/warc/402da48ae21596395b306cf42b9cd5c8f2b1d5a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928102.74/warc/79333d677287b8d2b877c0567037d40cb376ef04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928114.23/warc/5c1bc1b6c7fdd74282c0b0db874e0f670bfa0d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928114.23/warc/c6258318256d89233e1c04d144ee51a9531eea35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928114.23/warc/c68325ddeeca6a8981ac9b031ed1bf1e7249176e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928350.51/warc/282e547ff99e3a614b93a6a37d569965e35c5198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928350.51/warc/7183a4fb8a504d80a74a3b77ae865768ef88e3e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928414.45/warc/35946f3d43409b78de778a631bb1875a0b0e87b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928414.45/warc/521d5e7f6ab2d44ebdb6d3a45e4328e40dfb59f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928414.45/warc/d4c95c1dbb14f15ae6fa618759a8f3543020b99a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928423.12/warc/36848a0067f7dd2615e27570050dd2add38978ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928423.12/warc/38d0587471c597ca7057d160c880dae7dfd51f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928479.19/warc/225b0b19e9887ea38911ca9d58d142daa05ecee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928479.19/warc/50333b11b7e3fa8ad96ded7d9ada6744f9859748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928479.19/warc/e8e6fc656e84a5ba79bb5fe18732c194e5820fe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928486.86/warc/58582f62c0492a526e3f2e4a5df4fb7944f81589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928486.86/warc/9e7b87d1dace8cdd132e7db9df06aa1772022c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928501.75/warc/2b35f5bcb7f545a7a755306a024bbe30849501ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928501.75/warc/aaf1f890df269932e42dc44df10466172fd03a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928501.75/warc/f15708f7ebbea4278731ae650d61761a6b11e829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928520.68/warc/8ef810cabd5e7ae3a61a4b577414b4b58c5d1e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928520.68/warc/dbf497adf6b71d8cab333801c776b8e50b9d9c9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928520.68/warc/dcaf756ce0bcaa129fe177bfc7373db05d1d7b24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928562.33/warc/a7f2a4e3ac15af9256f895fc165c7ee65b243588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928562.33/warc/b95f8eb4a1bfce63e8afbc89f74c3d0fb9d34467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928586.49/warc/0740877f45c5d2628fdf08452d790226bb21e708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928586.49/warc/a47f3f69c39d50919c82f28d4a7ed6ac7eb96c28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928586.49/warc/bc9074303a3bb054abb89885e6e2606649ac5493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928715.47/warc/164c171890f973989f959125628b49c6c4344ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928715.47/warc/9faf197b9bc78b4d7c714b368a40f42150e3c125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928729.99/warc/344bcbb7c7935452960a6078c613208d0993693d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928729.99/warc/c0b9e823b6a2514933844866b5c3dbf7fc1e0254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928729.99/warc/dc90fe76a04667be85039d4d15971d70cbc67c82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928754.15/warc/9ede69b996acec9adc7e20a9d924615a8f51c5b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928754.15/warc/d56e06d235788e3e1d55ac66613657acc1408127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928757.11/warc/29e51f35dbd8fa0de0444763fe4538a14928ca44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928757.11/warc/8ef78514facddbbdbb8221a533875ec64fbe27c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928757.11/warc/c52918c8284c1caf72f2ad0206d0cb5eddac9380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928780.77/warc/4818b2302f7f63fd7d0d15b240e80af8fd6cbd7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928780.77/warc/8557c14590f9014504bc37071b15992c4f24cbf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928817.29/warc/26a11f77e895cace31df9fe8dce8b58de1d2d2ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928817.29/warc/2ea64f3e239bac48666f4b270f40955c17e64bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928817.29/warc/8866a8ea58910a2f8f5c17dbabee09cc5f78480e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928817.87/warc/62b5ee68cd1f89b3a7c7657ab1ef8b55e4679ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928817.87/warc/b525618722415d989f7c4ed58d22e6713961cce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928831.69/warc/4d04c86a7ad429c7916f607504c85427e20f1dc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928831.69/warc/651232eaf6002aaaeecc5aba19155594fce7c596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928864.16/warc/b288bf0abb329bb3cdbb322e056e6b50470c9fab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928864.16/warc/c7c3fbba0afd99f7acdeb48f32935e4d0c4c2950.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928864.16/warc/e510eb18634af04bbd1c167073aa6cfecf45a693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928864.73/warc/0693611f1c02cd6960a3566036cbf995155ee6ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928864.73/warc/4216db4c4570a85df1e992f57f983a59ecc32ecf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928865.24/warc/2f27bc5b1aa5292e4f041d37b164e0751f814691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928865.24/warc/6fd97804b4e4862314dad6dc9e552491f7a10fa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928865.24/warc/b39d7c054fc20d09a8e2b9b80b6734345d2cc717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928869.6/warc/b4c5c566511b4d1fe3c8214bd678c250f85209ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928869.6/warc/cf91ee2edf1dc526d7ede1de6f09e08f6453cd33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928907.65/warc/18ee159f8b9f00cfb8e4158a486002875338f46d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928907.65/warc/77ea1719e91e6a61abade482e20e788af513cd6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928907.65/warc/7e012a1cf7d1eb45337fa59a84f54d751d2e8f65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928923.21/warc/0b5fe1d6df5b0555c46506ac69001a2f3e2bae72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928923.21/warc/3ed8b0bcd83d193f53788a339087f475c9bcb565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928923.85/warc/a1a8656b81339c7771bd1e3e0fb8a25e0d790283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928923.85/warc/a7d5aa3cc84ca7312b988971e20b6db6f1680db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928923.85/warc/e1e7fc230f74cb4b18f71d079377e0a18cd98d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928965.67/warc/9ed6494fe4276b5a9100028df9e8945aedb3cd95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928965.67/warc/bf7fb39706f0813fb685d3aacd643789a113e7ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929003.41/warc/1f6f1e7a2bd4389b09d1195030e8c15444765c7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929003.41/warc/b899d2e7c8bacde5328fd78b2821caebe293785a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929003.41/warc/df8980aee00473dcca4856fa09dee4b70a1662a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929012.53/warc/3ad666e36ea002127f893d0d320ba63d84ea5df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929012.53/warc/6b4e259958b548d0ad14828cd573773baca73ec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929012.53/warc/a5efbf4b80d15f1cc6f878fb352c1109f562d0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929023.5/warc/4d981ac3c9a8b2e15babad5282df752ed4b6f57b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929023.5/warc/69729de8fbd2c17811bd86e2ba6428c03c307820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929096.44/warc/5c3a3e6a38f2095b6b708c8dd64014fcfc790ed2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929096.44/warc/6a81e4978baae059b5955cc840f784a27c4b792c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929096.44/warc/eb2c3d3baf68b184ada01591e2caa9b0efe429c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929171.55/warc/0d9b7b4c30ed5797ba65ec176d3186e1f3dcab0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929171.55/warc/1d630dbce69d985138f97da241f565a2f7ef0b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929176.79/warc/327f0bd23150bfcc97c70f00ebc1087129f0e0cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929176.79/warc/4e481a35fc83e560250cfab272018f8e75512156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929176.79/warc/cf5938b29d809dc7eafda1cac21275bf06f54e8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929205.63/warc/42838845481b8bc93d56547ca3ddb0942a6f5167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929205.63/warc/74460dc5de2916bb99e221b0c1b3d3b897bcdfe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929230.43/warc/4f776ea2325305928e0e85395a456acb5a6a2c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929230.43/warc/bf53c61db7e17ec96574a9d264883ecb29bfba42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929230.43/warc/ec43d42c8ee843044e1c05a5ed3a91cc3c33143f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929256.27/warc/9d0fdabda41224313a67f30eaea6cb85b83b8b66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929256.27/warc/d97e0579d0fcb02dc4ed7b0f59d698137c579f66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929272.10/warc/2e026084bc70054f3d6e4911d7542fea13262ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929272.10/warc/5c016b82ab3fa015b4f7da298c5544bfe602c79c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929272.10/warc/f48d06c69a1038238501fca8dec022bad4f5b1b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929272.95/warc/1bf08e82930c925309835ba74b8563ccd6d6fa77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929272.95/warc/8f41e438c404bcd2c00ec2c904ee99239e3a7c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929411.0/warc/7b9bb2dc85db09cbe8ca87b9520c81d079e0c088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929411.0/warc/a7ea1cadf228758483e4e5fd3d824f62b6a769e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929418.92/warc/ab06a32b46e80190a373df0f573b612abb2ed02a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929418.92/warc/b4558a99e4e9a2f269d6d3f5dce4821aee127953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929418.92/warc/b518e124d5747f7812b154864f1e7e4fe926f856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929422.8/warc/649757b628b6b952fd029ab004760feab2e05a7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929422.8/warc/66785ce491b0fad9e72ae7a733a2d6052ceee409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929561.80/warc/54d949a5fb3dbdc077d0231b5460d84df7c76684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929561.80/warc/917b689110c0f3aa931df8469324e13e46afb796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929561.80/warc/a574a71246904df70ef403dc50819ac4942d9d4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929561.98/warc/e26359a30e91c68b92f772c094565bddc6069a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929561.98/warc/ecc8dc3070d203c4dcb301fa4a57dcf591db435c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929656.4/warc/22a48e3ba6590c21227f21df53d0273cfdd6f510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929656.4/warc/5c6fc4e7b1af7c37e01483b70985ed739ccef78c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929656.4/warc/688ba20cd57b55f63999670e2c9addddf28ab503.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929803.61/warc/9b604313bde684b913cec6f5ab57a5f533566d80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929803.61/warc/9d931ecdcf78e44ce5dc56939d7520ce28cbba6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929832.32/warc/0d5b6b55692d5f8cd0170e5000fe1a9e7df820d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929832.32/warc/2ba01981a665d4126edda8af9554396be8c72beb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929832.32/warc/c99648f11d42f60988356badb5ff222bd02cf6a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929869.17/warc/5bd35b2b6a7fb6ed2abafea68cf0a52f1e7ce36f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929869.17/warc/a0e88d9a09e4f531ad8e77df631ad8ed2bd1c142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929899.62/warc/5df85c6182a89df695122170bc64201b2c396aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929899.62/warc/6877751c3b76b6c242525245a2d8ac7874cc5047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929899.62/warc/f6275228ae8f969a4a3b88f130ed4196e64b2a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929956.54/warc/1090955e010986d45b85d4d252e969c8c842c3f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929956.54/warc/39bba298476e414dced809683d6893fc5c0d9186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929956.54/warc/b384484600d9a9557c991ce7ae9cb0249c1bd819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929978.35/warc/20eeca009bc32e26ba1f0d8b5de48b24ebef27bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929978.35/warc/ebd1a883012398600c606b57751177d7c5219751.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930109.71/warc/10b331d4d28465f5c230192a7d9bebdabfeae1a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930109.71/warc/121aaaf4481d3239a27182c604a24b842124d18a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930109.71/warc/71d99b9799919d238a8b615ece418ea7de299b4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930143.90/warc/3750a9b377aa6afeae6ba553ca8eceb41492675a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930143.90/warc/4b66d90e8b4732f3df0b61b2fc630aeb6ce0ffad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930256.3/warc/169e8f8142da48676eea5f86b60e02b8aecd5f3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930256.3/warc/1c37aafdf16322f2372bc590d1165760174923d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930256.3/warc/9e17d76f000a2023d4609609ca18c575c8837ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930259.97/warc/81eb6ec33bea4042e34e6628a37c3a6cc958a21b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930259.97/warc/92e901cebace73e443249fe3b30887fba6bb48b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930423.94/warc/3bf34996e6d948d158fc3837c15c82c31dbdc167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930423.94/warc/925638e664149e8ad5d4ecd8f608d234451db232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930423.94/warc/e2457d177323d492e9ed0b4b5cc56d982953d99a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930443.64/warc/798e9b914c5e53d87ab659b27c8425d02c331374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930443.64/warc/7a12ff147b91f70c80d0c362552e6528aece95f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930866.20/warc/2086a222468f82133600311998a17d2119c2e398.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930866.20/warc/41d70a0c0d3288045af7c4b28df3c3e31827312a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930866.20/warc/6823a8a49b25e8c2eb5d63d87b661508a53cee3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930866.66/warc/7b011e441baba6bb51b43c09dcd91dbb5197aa8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930866.66/warc/ae756e2f4f926e1fd0dc286f2910962e6a47f244.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930895.88/warc/21e6581896261a8779fa3d8d9c328917e60f283f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930895.88/warc/e762e0099582fd9ed8a0378435c67e9761079c7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930895.96/warc/563a5d3b774fb6e41652cf405445ddfa886877fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930895.96/warc/9c85d77764ff6cca04dfab29d233b4b29c387251.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930895.96/warc/f7d6370d250ad8a12395339f57f779330b3431a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930916.2/warc/2af066eafd175ef14c57d7f18c92b13d318814b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930916.2/warc/525d24553d77a8ca02f166646b7c8ee8366e12c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930995.36/warc/0c8a02969c0d7f94303096f3e576df386f949689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930995.36/warc/320cd2d5dee698297a549b2f9e171cd5c6917037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930995.36/warc/61178a90a70c814e0d03008b6059e856392d8532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207931085.38/warc/2c0677348e751a5d47c2d85129db9b38e55973a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207931085.38/warc/d3b6edd62cf4dfcf7314755c4044811873145de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932182.13/warc/3e820782c47896d92b64f24d9aabbd9fe088837e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932182.13/warc/c895b37957f2b3d6a9ccf4496c13b1c7c62203a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932182.13/warc/eeb4ecf07105a017a0a2d560e3842cff86bc321c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932182.89/warc/d7f2d928d06e6b0f63e63604259649a2b69a2309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932182.89/warc/ed877de5b34dfe1c8cbc4c86381e455f585bdf32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932596.84/warc/0a8a7e698ce180732e4be81840aeb0b783fe3c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932596.84/warc/3a33200d52c96c651d3841edfc426db38de45174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932596.84/warc/b0a68bd1e9afd1faeac6fc014f7846972a09ac94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932705.91/warc/e95fb72f9e8b6de95ca48c249d7c9c3d8263d758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932705.91/warc/f5fc36394a37dbfbba5f05d6b575f4323d6b3f5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932737.93/warc/a883d05bb79311d6b0eeb4b01fe0d2703980bf9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932737.93/warc/afcfeab5db21069b5183df6d6a3d82f0742a674e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932737.93/warc/c7daa7814fcb5e9d15aab0c40a36ab986f7d141c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1433195034286.17/warc/de8773703035c4d646caf45a9adc85169cedd032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1433195036112.6/warc/ff65ecd1a01c1147323d518a57f9eb944e2164e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1433195036702.22/warc/f755adccc0bff1556665f2dba59a57a9ae02ed31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1433195037030.16/warc/faa4903d46dc2623cb57080fcbe9e42bc21866a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1683a84d07647090d26a41a19669b9c1c6a849bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1996992e75e2d67b58e7b836be991cbf34c50f71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/22fa57642331b427bf641cd625e0316435a0dc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/24b0c926b749a0c6bf9ea5ac01f0545cbbdcdffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/26173b572dab7d6f39fd7374e13e2a266b118fd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/2659986af84aa88fb5d1d3b2e5213d35f095373f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/28595089cc12490dac3b8915348fc790d8e57c19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/2a833bfb76c216948111c2c75840b09076361517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/2e14ffa1d03ea34d0e942faf7119e22634aab244.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/30907782e23eba620794babe47375282bfe6e065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/3340aa305143216ee92da9d730a228578ab0618f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/350621942ac6b3b013a8484cf5beda31ea4e4993.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/35ae720b7dc032df560b7dc2cfc198724b06f57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/37c31126cba90579037486c89be35e8a640689c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/37dcc2d68df2b1b97b32b348ab88fe8e07c9237d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/3a6a60c513e8f9adc2bc177d207190f1acfcc2ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/3d653011d0dde4ea6915adc36ee931f5063205e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/404d7f439afe3ee7fe478ee9499a8152889b2699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/462525aff50182e9241a13832c92607edf4da1a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/463f6ff68d5c8a340ae616b90a4707aa194bc75e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/466e14b075751861b5984cbd76a11cf8fb7f1307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/47812d6a3eb2c968fb5a5fa9213c0ab500b25c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/4a86f5f8d5a02abc34aa4b4ea71d742e7e188fd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/527270919fd86a4ddce2f1478f6b02eff731e531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/55302c30c97b9fce5805a8090aa004f0637771a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/5f840106edb11d428c3f7e703b3ddd74313b3452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6438bce1ea54f6d002780486e0bfd5d3df985bfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/66777878b87c2302084b17b62dde5973ec186da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6695eeb99c97d942ccfbce53f82f10c8830bf281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/696ff14eb0366d22181af340f70469afe2ef4c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/69d0e5c4820047692b684b9a736c0c0b1b125ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/69ff27c8b407b05af35acbf20c139c6d9004d7d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6aa353daf78967ece7507fc3b1d69f0a7f26d2b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6b9fbf0f87e50df6d6e538345266c49430779e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6c027f2ed2ad73b5959351fb66af13b78e386a9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6c3b7061e82db7862cc74f4f45cd25e02d012f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6cde3a23b58d4a72eb714c30d5ae7b6a2acf68f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6f99d18c04a8f3edfe1488ba74b2cbdc76e079a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/70a004ac1a8ed0cba79d949d72f3f6a0f48fb7ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/70ff50ce2c1ac093edf3df49f495d748ef12ec45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/7545dfce38a53be126809f06c644bb4fb91a19c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/75d424f7fd8c2742585f108be990651ab30ff02b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/7a7e9448e6e8fb51b371d4502177286ac1a00f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/7c23f8defce90d037c3bb916ff2f38631103996a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/7fca4c63cb0bb28aed7d16b86d96550bba8619f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/7ffb606e976f4fa11674b7a91cda4a3f4a41b553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/8268e8d8a3d418e38fd3a6863cdff6d2c4a5665e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/847b37fb58ae44acad204f6ddb26af0d10b435ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/8b8a7cb74969b82b64a02c034e32cf7cc835ae1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/8e0484dcdee9750fa957e9c050787c4fb12338bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/8f1f8664a8ae82dcd8ba0db5fd0ac94c7111df2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/933aea1c92ba0ab70bf801a05c0f90007e26de27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/93ec6d3314f46a702390ba3598bf61864a06f2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/949c4ce689bb701532e1b915a935b24b1e6650b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/950859a60063b7e62b7985aae7af5b8e689b8bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/977c0475f5793410bdd3b5d38e809979a54ea8cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/9d5e0644fb3976b61251d9f0e2f8052e9bb601da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/9e7856562f62a46ab5605ee48c2cbe1a56820719.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a06a860b78de409f9eba532af07504c903af557b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a1904a48b82a3a3d469c4384ca9c12b83705cb37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a40a5090d1f708c52db0c1427c1a7c057e29727d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a41eeef83a67f1411ed5faae6a065250d29e102f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a7670ced250d105e292b19dde48faddca67d8527.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a7e73e85325b4272b2624c5f5b45c8b39f2977d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a7f7a3126d0c503c5df08d600eadeae0131f2474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a9a14ee3bee9cd7e3a0e8f3aba787d7ece5b30e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a9fd00a0d55fa6a093857ae808763d75416fd143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/aa3e50b7f562e4f35be0115cc8aaf96ca566fcf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/ae3f3e23b2dcb135faa32e2eed7e4ea85b667499.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/b1b8e3731ed52b0350a1405e0f02047b5bd11e6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/b60f3cc03015c1faee1014c568d9a19108664caf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/b7e7aefa0a422c14f9210b5fdf9f479f6e3dd667.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/b9bbf61b04d60aecdf73df02d0e41e01d3914436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/c1b004686336d614a635608d2d36c7ec4889e304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/c4b338e9a96bc7676c14c86fef274d2e557692e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/ca0dcc53c9034a1d585681119400279b07d6b64d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/cac6a525749b783638a96755e3f57e8d58a5b935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/ce8befe68f21ac9711678ce98df8d1ddd153647b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/cf01ea6bd5adade4afd0c5e7d6b2a2b2cd06095c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d00e6e4538e2a6aff10bf53dfaadb7c6451948d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d14c03ce48c323d5b6d842de645d466b72fc29c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d477f09798cad2a2070f61e20034d7ead56c6e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d49140f567c75e60e55cebcdfde27367ab7e7d0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d715364b56a05f93f03215cd7ff300ba4a39a91c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d73d0223d8377c6647c2c91d6af63ac2ac1ea9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/de62338d9963ff6910a2b66db425cd014f501a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/de707310659b56c3915c2a09eb10503bda7b91ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/e45708721206ec2780e7d4c0b369ca1b30f8b75e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/e463c51b5163446753d97145c166c1be33a130d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/e7f0dd50ebbdbb4ae077afe6960442d9434db977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/e8ee6dbae141b181206bd2c8fd32cfb002b49ea6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/eac1ff08657f1a17607b14531af6efa6fd6da977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/eacd69c052d0420fa2151e23e0080f9e33b80cd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/ee1daac8f77b831275a6b8f09abfc921b5c33ba1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/effeaafde72cfa1c86f39f9a098b6868de003adc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f034bac21987537f7de3b150b7bc9f082fdc7abd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f28e3d2a57c318f36f6c663b361bea7fc51e8793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f337ca711f8659688b473cac487794364dc51e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f5d9a3356dd92b06dbd62121b3e2f6552376f8b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f76ec1c72a5a9aa87ac96bc1ffb4fc2711dfb415.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f997ab1ab76eb5364452a55334e244ee7d1130bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/fa49b8294fefb282c0ceb5587a5bcefbb9693044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/fff85ba2f0f3e94d384227bed7873708347c5509.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2015-22/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2015-22/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2015-22/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2015-27.yaml b/configs/cccc/dedupe-url/CC-MAIN-2015-27.yaml new file mode 100644 index 00000000..476bb2e0 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2015-27.yaml @@ -0,0 +1,331 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/00a8b03375316a4fbbf4f1b9c7b0ef32a2dd0366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/08ef7fa74c0df837fd7db18814c7a2c5efd2177e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/0ba99886d3c1021fa8cb7a393143fbc9937b5374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/0d493ee1d154dafb683f82c13b0baa7e0c4ee149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/12ecc5f2a72d9f0f29b1fd7206d35db56e18e161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375090820.0/warc/2f1be7414feaf0eddc1973ca2f3a47b9948cafb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375090820.0/warc/5c601a9935fce92105e6003ec4d2e4a6b54106b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375090820.0/warc/cafcdba8f216e5c1993e5cde1bc2a599fc0aa1f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375090887.26/warc/0edb919d0c4e7f00b3fa196d29e864158d28a435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375090887.26/warc/3e6d35991a6d68f5cd83d3f447a06a06a9e8a9d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091585.27/warc/8dd3a25e84a17aec1cac94460baf3d0d84d0368a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091585.27/warc/a6df9e22c15a9826eb8572c7048683d12137aa61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091587.3/warc/8208e81835f2570ad40090f03f651d50cffa837f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091587.3/warc/e3482e36f2df6f591484beb81a1fd9df1a98aa03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091751.85/warc/8e84159054866146b424dec7a93d1bd747f9e5bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091751.85/warc/c40c7f4cbfd4700215b78dc81302af26d6e12fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091925.14/warc/1aec0cc8a0c90e1f33bc2d6f9214a731b4b6bdf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091925.14/warc/838a994ad1c6f3503ddd6e7d092dbcc9e430b9de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093400.45/warc/c65d4a77e7cc25bbed4b8f2ae977aa5f5bd98801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093400.45/warc/e1c3dc62c1b9416aa9bfd848a591394f6c67310e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093899.18/warc/4d00c772e2becf134b7a7201700f0472b70274ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093899.18/warc/fedb848d17f1002edd6e000cc7b66259c1f35601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093974.67/warc/07696fe96ca5a7149502f8379b2160f1073a3359.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093974.67/warc/084fbd7d258cb77921d282b6d2ed7694d9fd41c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094451.94/warc/1e2ebac6aa4c40a20f7754036c6840a8ae4157d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094451.94/warc/6eb7ab13c5707e7de710c45b2d9d1f5a03e75cdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094451.94/warc/e4bd5078b8f12cb765698bd7eeb5759339c9a69d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094491.62/warc/a775618a8c7877772010cdd95a8c12553a8560c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094491.62/warc/cd4308a92c45d64a5c6763b5e2f3bde268054ad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094491.62/warc/e57bc7ddbbe2b0decccc92ac8ccd4d23263c22b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094501.77/warc/63b3de8f19d006d310afe1b33525878a436ade64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094501.77/warc/e22af352c8d403f4c7a3de1e1562a4d45488ef1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094510.84/warc/73e4b5152cca7ce49b016d1a2f90fb13e670168a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094510.84/warc/8fba039e9f968ed41033353565bc7d9336f7f6a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094629.80/warc/26f6a5e97a9e129410a708dfedb3bbd4d4e71635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094629.80/warc/3d7ebd6d413e8b550adb6f00415fadc4d34c4f37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094634.87/warc/19aa5512ef5d939fe277f2412f81dbd38e80bc9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094634.87/warc/bf08ca7547c9c2f60fa7fdf76878e1885d001789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094662.41/warc/625ff8563f0eadf04fdc7c05d37a400ed69d5604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094662.41/warc/9b8bcb110c1b0a7bdbd08fb4c070c36efee0c8f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094690.4/warc/6c2bf4ef482d40362aca60df5b48443b9b07cbd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094690.4/warc/d6ca0f595b69aeab40f28b92acacd0d26e448776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094735.50/warc/f918868c027864ec4e8821ef998c6f5d96f85ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094735.50/warc/fca3a2062b54615f6d5e58adec9264f664f83107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094924.48/warc/5764155757337779aad0554cf7930a6a0d5544b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094924.48/warc/8299bbdd6e9d4d12cc2a118bb807629348d4efd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094931.19/warc/1903d361b978e23b3fa3a241d04c8e5ba614d02e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094931.19/warc/2ae83d05bc01ed18a4ebc03747af80def66e0f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094931.19/warc/ac3c88e888422de9a81370a68336f64baae45457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094957.74/warc/c74502b422cd8220807ca22afd3301fe9c228b69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094957.74/warc/ee0d478307c20e06cc69076721cff8b169cab863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094957.74/warc/f20b78a32844f5318a7d8cc89604517903db95ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095183.13/warc/5ce70835cde49a505f1b820cb4036bf23f4cceb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095183.13/warc/c99830168e0f355c769c51a6b1b7237b334ca453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095184.64/warc/afb639d81fe23295de84f5a26ac472f9f5d99d98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095184.64/warc/c792da577830e0f22121d7477cd156cd71f95e6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095270.70/warc/5f1efe2be292ebcce009fb92956b6ddaa9943d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095270.70/warc/84a9f4c79f1f42e58e2fe6791e9d5101f2c9503f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095273.5/warc/40e50fd11ef7353ecbe8655db6bc75dbe92fbc52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095273.5/warc/c98dc8a3763dda3775df813c6458d76479cb9422.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095346.56/warc/59706ed603f70b83e2570d622ac21af0187eb724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095346.56/warc/e88565684b23d08e9d08791f5b8f82daf126e37f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095373.99/warc/b144f36daecdd2cdcb00cf35f0702831743c2e5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095373.99/warc/f15f954488bdbeb9a7cc7552bab2992a6562d885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095404.78/warc/03ffe8fbb3e50de18cf1fd8616d189e947a284f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095404.78/warc/de7ecf826980a2bbff8f750b697402dc85a6ea07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095423.29/warc/2bc3189475b3fb061c0feca3a52b0e5113c038a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095423.29/warc/2eb57f45c08caf30fa874e71775aef919687c7c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095494.6/warc/1ce6f8d853f0b4218722fbdc54734b860d7e10b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095494.6/warc/30be5ec0b60ff834ce2f54b0136850b4e5f21419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095494.6/warc/f394a79478156f5580aace1207fc7d47f89aeebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095557.73/warc/a95f674c20225b42fdeda1c59fa7402a37196ea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095557.73/warc/b33921f5ee3dafcf6e3eed14b6578725ed60d230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095557.73/warc/b355f7d070c819e98b8c8091329902099096962f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095605.30/warc/1e97783d2c835097137c7a7fac447c9968aaf424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095605.30/warc/ffd26baabf9d129dfb149b93088b8fa85037b8c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095632.21/warc/33356904958d606eef90d17a7294a0b8d91996df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095632.21/warc/c05c85231d2f723cdee8e3aead17ff833603d51c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095668.34/warc/057be0614265884de9a218008ffac4cb45fab45b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095668.34/warc/8c8bf2e8098e57acacfc4d165966dd2ffc7401c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095671.53/warc/27d6061fc1306215114f877f0ebf54f8a80ce72b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095671.53/warc/efeebedaafc5b4400be7e31e741805ba8e405647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095677.90/warc/afee2566d1ae190b570b61add8128350fa6c9cd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095677.90/warc/feaaf29d8d00abdb2dbd5d245b36bed62cedafc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095711.51/warc/7feced0c3f07c09ce1af4b7b579a108c891a962d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095711.51/warc/f0269183c12b3493dbe0babdababeb11f5406403.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095732.23/warc/616dc894808f4131ddc884ca1005b0a846240b81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095732.23/warc/fb61c1f1ddb815368feea4a200c7b49b554bdd81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095775.68/warc/dbdcab78634772b2d11161b75dd50be957422a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095775.68/warc/fdc29f6c475848980d948d6864ce5ce90d5aafce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095806.49/warc/5f0f4caf9585dd437d879999902f864d5ec8f922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095806.49/warc/86f096ca40253736b7db244d915b9d3cf881debc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095806.49/warc/fead607f71077b30b03d5e95be6b7609076579ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095874.61/warc/0a9741a796398711deeb5a04a5edab254ca7f8e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095874.61/warc/40e6ce0b24426b52d53c074e1e1eaa90f86d92d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095874.61/warc/7023e25f45dceb8c8abfd0c4ae74dc0c040a7f33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096061.71/warc/5f52cc075bf77e7abd5a0a63112a230ffdd2fcab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096061.71/warc/7aab8c50304be054504961a91623e75e2d6e0b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096156.35/warc/9b912d86fd79b728e2d1fee2f10e82c9c989b902.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096156.35/warc/c379df4505103d41763f35e40b28236ec8b4998d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096208.17/warc/10706ce2f6da2c321674c38f6e98d61865403dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096208.17/warc/ccb3cec1deb8c994f1af5e1b33819643b4243863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096209.79/warc/3aa9924b5388ed8c11f02e42f558b59bee14f27b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096209.79/warc/5c72852b318da244b63446626cd6ae18f21b824a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096287.97/warc/475b0babf911572ff249fc42cf9b7605292489f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096287.97/warc/edd0fea0ad471695d49e6f10712e3b7b39b26b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096290.39/warc/1b1807062e511a1e4b71db9e0e539f7a81eef082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096290.39/warc/c935adf7ad0d0fd9d3d240990808a20cac3af786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096293.72/warc/7367df5762a878b4bed41caf4c19f0cb00220eb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096293.72/warc/7abdb88e468f67b1fc0f79ef6eff308b22e19a9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096301.47/warc/3c723625ac393649c08785f32013975dcb35cbc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096301.47/warc/e70e3bc9689cb2305bdf48025a23aa403b8368c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096504.12/warc/25d108f0a11274fc843d8abd38df6bec83194e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096504.12/warc/39c23a653bcc70e4a4c296d65aeeaf506c80cc15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096504.12/warc/ed0094fb4638c5925ccd411d7bbb292200d919ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096579.52/warc/07311632682df0db39be1a6978691fbd81e1200e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096579.52/warc/a6839d3288901b99b558846bf849ac95cfbf0149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096579.52/warc/e9f091c9dc1c7a0fe85e946dc043014b38cd4a10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096579.89/warc/6db7cf7c1ceecb54ce1704e9ed17919bd98f0e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096579.89/warc/97074539898a70996fa5cfdf58d2c3cebe797edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096686.2/warc/082bbd2bfff03477cf337f66713f862ffcc8a3cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096686.2/warc/c47ef3738c2beee25182aa4384973b3d410d77c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096706.9/warc/3e16267a068ea39d20d8ddf8ac323b6ee10a87ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096706.9/warc/5fa5427af850c5476aca51e4571745693455a8e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096738.25/warc/4fda5f1e78190a6b0182632b2fd548582f16c6ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096738.25/warc/e3d84f3ed5eca5e874be760b73a5bd76c9a60668.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096773.65/warc/42cc5625492842b0a07c653eebb571ffb3fdf766.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096773.65/warc/510dd346ede0dac0c2f92a289b294b8d7b86859a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096780.24/warc/21621d96a409f590e35320c1aebf336ce0d968ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096780.24/warc/ac4749d2ee7a0a7e3eaee839af45de20010c458d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096870.66/warc/97db6f03e4089b5ae7445c39667f064eecc4bfda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096870.66/warc/9d15a2babf46e0c78500b5a6e6da26abd4e9c3e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096944.75/warc/6569bc32b51910112304664ca5560922e3b4d13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096944.75/warc/cdf8bfb25111a30b949cdd58b493908f352286b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096991.38/warc/2b0a49007a8b0b066cad61ad1238f2294458ce38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096991.38/warc/9fa4df7df30a5cbe5aef8a57cfa5a5dd7c6ed520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096991.38/warc/c7ac8a0cda463a31a3fa4d3251c4b2b79b20399d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097038.44/warc/358337053388c03679f68412ba6971d4c17d025c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097038.44/warc/5748440bfa1ecd38552eeb910c441a022bb11e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097038.44/warc/80ebb80bd71dee47a3836281086121dab64e01e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097199.58/warc/011583bb1ac02d4a8b3707a54cabaf8c15f09cdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097199.58/warc/ac076d1dfba9f4551e135f433db4182b2d5d4b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097204.8/warc/b6c1e390bf8d98b4423ebdfee4be2aaf708d1923.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097204.8/warc/eb2929b0dd34f7364d54c77c1622c5fa6e87aeab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097246.96/warc/3e13d11f772cb9895cea1f9fbe6f628da177b523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097246.96/warc/50c5bdf1f78cde653db1f1447aae7e6790d5bbed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097354.86/warc/77cad681c44136b2f7d88dd405b17392d2a9b3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097354.86/warc/c204ef10d2a50deb81bd98063263b89f4135d153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097396.10/warc/0fa65598a255103d49982dc02b2938a91153e6c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097396.10/warc/f775785fc0be1911ff703e285693044ec86f9b59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097473.95/warc/313b0791e1263cf8f5e418ca0a29905a730a3b9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097473.95/warc/9bf84ef58b5c5d3c445ec2882ab3e955b97357d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097475.46/warc/241800568cd87363fe802b1ceeee68034725966c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097475.46/warc/5f2d38e0ca27a1d102f415d5366419815a967dfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097512.42/warc/14dff3885d84f0e16b6bbc2b7eeefc24718b26ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097512.42/warc/2f604569bdfa6f705cd47e84ac2b779096a6f39c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097546.28/warc/2d071d44f21b0addfd120c3b592a76f3bd76dbc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097546.28/warc/854569f0bf063043a171429e11ffca6cb5d24d8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097546.28/warc/bda04f23be0431783300d7161e4165787d06b062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097548.7/warc/1cc9aad31e7a8f127e3543b53581db8102e327f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097548.7/warc/3cac9a945058be583a451903eb292243df5ca530.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097548.7/warc/f3cdea56a81c1b2d6ad0debf31cae1462c99ecc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097710.32/warc/01d1854cbcfd1f1981134369621e61abdc305178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097710.32/warc/cee9a3dae2a32f775cbc0f89f760ae8b78849289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097710.59/warc/01f3294088725964bf31866a8598c48703e1b227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097710.59/warc/8e319fbfe3a55b5684f059d73aec29fca70b9230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097730.92/warc/00d515b1a406a8e9a77cf24b2465961affd0aa4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097730.92/warc/4825a30d352b677dc24e64e65554e876b8874abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097757.36/warc/1836f8a7e320719acf302cfb9a1271dbe11a8cda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097757.36/warc/c3d12b4b88c3e7a66389ce4379a9442d6e42f89b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097861.54/warc/70aab1e2a6cdf49bda9b7114e300a3ba16241783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097861.54/warc/87269530a6dd3e9e444df9cee665b3a3c0b5420b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098059.60/warc/cbe795d74b09846011a28eb073d51ef64de90bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098059.60/warc/fea238db2644c393520c0bd8e2d8c26f6545ea20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098071.98/warc/bb18bd0421434dd65b8a78abf486320cc339f440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098071.98/warc/ff9d2c70f83ddd4157dd1d134261c18d9de76112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098072.11/warc/93102066cea6ec8094657058c9547a1199584b43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098072.11/warc/b96833fd9244f57b2843161b5f608b1375aa3e14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098196.31/warc/00b688e62c0629e58480db2ad8f66b60a29fb3a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098196.31/warc/5edcc67eef63bfd48428c2d57218e129bb2fc2ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098196.31/warc/7477efd1c336f29f220a34766ab002cb14624f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098464.55/warc/1f8ab93110b01dd8dc24caa7ee1b9e1badde4de6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098464.55/warc/210d0c0df0033963f4d5dddbf334827918c1a8d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098464.55/warc/cba5a3b2f7cdb5eab211699211b494721ccf7cd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098468.93/warc/dcd2cf435fe0fd26849c5dfa1ba7cf01c6077e78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098468.93/warc/e84451a3851c6389eeee13a07c302068f5a6698d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098685.20/warc/9ba127c1e6c5faf6e3c9688c5eaa0efd4cd52d05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098685.20/warc/fb19e77e0d79440e7879b56c65e94166099840d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098808.88/warc/373a217e6f4f1901260520bd7611e8b622b742ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098808.88/warc/dcaf99363057fea9274df2a69fe953c726e0cd0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098849.37/warc/1e2561256cea55b261dc581e3ec05f7691397a4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098849.37/warc/be00713862e7abe676e42b44a20ed8bbdab9c80e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098924.1/warc/85cd7e3cc1399e795c3b6c5bb7980cc75b938816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098924.1/warc/9f73f8cfab9b2be1c66744a248fe6953684e9b31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098987.83/warc/0f1916392af4c9012d407b422fcc588e765b5064.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098987.83/warc/c4d5a84ca5b82e66ed20a6743ae5b3d66090ae82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098990.43/warc/09cb761ff679fd0b5087745b5b7b57941721b5ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098990.43/warc/c840ff1d71a717a4ed87334c1724e06f15f2fe77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099036.81/warc/5cdcc74b5810600269ae080f7d91e1b399324ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099036.81/warc/81f637b09ecfca3c8777652a0435da92d2d297e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099105.15/warc/38da14a9bda59e65568b2425b57dcfb4ff1eccdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099105.15/warc/3fe64e7f5213360af1d3daf9476f8865fe5e9d44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099105.15/warc/aa4febd66ed6da9034af5c25903d9c03b68d9ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099173.16/warc/9b9a1056d77e7c4575a16f17a9eb950b081f55dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099173.16/warc/abfbae6d43081273fad62de1ef310f40172510e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099173.16/warc/cb6264fa3e4ed32e3c2fb612ddd2b6d5e850c7c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099361.57/warc/47f68eb66835cf5563c6a198633243dad06de3d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099361.57/warc/70c99d97968435f236336283bcb005284d73e0f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099755.63/warc/0da7e0d74c135ef9bfb66f384bedda7c96bedf11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099755.63/warc/ab2c96470521a53f5df12d5bc73547b236021c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099758.82/warc/50119068fe9da08cb2dde0c08fdf0581c426e99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099758.82/warc/942a84934a69a9c9e1c1f86bbdc8d1d5bdece319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375100481.40/warc/cc01dd2aaf8a7db514ed5afaecb8501e82c0e863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375100481.40/warc/e4a31667868903329cc3fa5f2afecc8908a5fb58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375102712.76/warc/03c3ae67f84a13ec33e61047dbd14c7628e0bb73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375102712.76/warc/04abd0771aa8000172dcfb93babcc298988b7c76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375635143.91/warc/146aeb8b1db11c90401fff040c8b98d321b1ff51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375635143.91/warc/da1978e2e6b627338fea5efdb40e0da789c3e31e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375635604.22/warc/2e69c86e875d8a6b3c431dbb8cd24ba727aab389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375635604.22/warc/e53542cc8935f6ea829ebc1bdeb7d286bde3dad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435376073161.33/warc/bf6b585f7257a07a847611f1b49b9716dbe7ea5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435376073161.33/warc/f9893310337af590694d87ab5403615a60af9695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435376093097.69/warc/22fbabdfbe7861457adb323d03b81acb575ea60b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435376093097.69/warc/3a86a101570e9997b3fd54d9e6937af67b21d674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435376093097.69/warc/c4dd11d0713d11c122c90043ef44520179a7e34b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/159b00224a11ad11889dbf7f0f2aa64dfd97ef0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1796667638bea4b11648ed2726b4a6ce0bc0d878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/19556971283317b5c8f87fcb9c8359dda458def0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1afe294708e3b2fa325070fe5a8411c7984542ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/21242b372b96a58713c1a06380cdf4c1062e4d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/21e9ae538dfadbea8a3ea8ad16d20c4d5cf6da94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/2433aa923fd9bd40c4fd09ea1f7e62b1ab532206.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/2e41d8f1535d2c1d789bc057d2825c596e3864e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/30dba691618f3010d9258c0c7f2d7f727aa8e69c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/33808794840aa7b416aef1928c09ed2d98f3df6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/3a29f3997c9facfb7138ea4639ec554a63e2b492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/420eaceb62a0ab5ea91008aae4259446544fee04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/427ba2dd6271cceb6a5f73a42a166ee208ddc190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/4eb32d79f8e7a1e5ec1240e515b9a10df5028e32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/4f2d8fba79cf4dada476c70e713c44c9fd150cf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/4ffb7e98d17235b1d10fc74455dff024ca69ed58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/5166fbdf4fdb7ab6ea621f736f4f8a2b27bc4d30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/52c0647d26aa45c8f42aa75fc8d2a009840cd296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/545b0664dad3477ffa669819f592599bbedbda92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/56ec6458706f829aa417d85d53523e251925d4ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/56fef31053f56e824080ee9e95b1687818f8d0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/57bc20af9c2d251464a5b57b485ec64864786cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/58dceadc6577a408ccf08fd6255a062b14efbdb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/597df5cee554f681ba284f31fb20f8b2d05859f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/5d5600dee511a6c89ce9330b5ab7229f29b20351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/5d5dcdc7fb8fd98791d69bfe8c905e20360c36a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/5f1d63f5d363c5b3a066721b5247589e071cb7bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/5f99f3445527e666b8e38f1b09e9f01fd3484eec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/63c3ac5b7b326720df5e85764b43659d0060b908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/64150726802da39a7c8c6ddf8cbc05cb28e6ba52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/6465374b6d4fced4536b7f78c42b90be98bb6432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/690cfc4de9d7d146fdfe01ec6d130c86cdc9c0b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/6cc432002bdd21eeb1c40bc60c4708ed0f298f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/71d18c37e0f72091fe22996cb494b2c3f9572f46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/75c35af07096a2459fc9254dbb555bb3e318880a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/7623a1370193d6294637f2876febf2663a4a4c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/7d9537d3952446d137ec5f256214ee8247e26588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/7e29cbc0fb26b096c23ebf6c67a9b1e764bab633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/7eb1c26d848960dd9379f29a7f7674609aaa3540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/808e8aef8b606e86cc73d8cc8ffae598a18c13ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/8102c0fc412b4cce9e814c1c562bae6dbb276c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/8179bd22262e01408dce5a3bdb85e6c7f2c91fa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/83f6021a49f6496df326188bad40932c729b4693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/84409d7ec9e90fb82f72b36991cdbd7ebd320d15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/8906b08694302b5f6ded43304b4d69fdec402246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/8b67d3be306f7f29d508baf5220bb554c2a0f13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/91056cca751c751336fc525c2f850a1c9dc6c252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/92d7d5c1c4536fee5351047995555b44c508e8e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/940b0e4294a27c25778e11b52fbe7239ee7defe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/96fd0b3267ab6a1b0c7f0afb14d2c01601596003.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/978ca04d27ca0c7b93a8881e9c1c6214abc9b04a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/999003b88395f4129dd899fdd48d1d0798f67b71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/9e57f44c30fa945b58fa2bba8fd9eb8807d50940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/9f3f3258c374a050ea2377ff2eca968049a4c9a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/a561a748277dc491c11ea8723fdac1e03ea2970d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/a9aec25ea8bf7dd36cc2685a545dd17a639c0dc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/ab1e113ee81d5a89e634f334f20aa481ba296a90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/b4fcdf528bd0db6f93914539acb8012aace5d331.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/b56c198784a149c1f5ad5756a6d9eb0c0f6b3359.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/b5b27be865cb33929627125072b9d85826de1148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/b7c5bd4e5458ec2cab79e912b44fe410d7e92157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c10133d024fd87d1e081d8ec8789e8d77eac8247.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c16de52a8c3e4dc074e4016c5ba2547553ecb163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c57c361fb80b283af1a7a40e76db4152a1075a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c5a166539f9083982d733797f80a5a143601e8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c7c0987913e1ef903a09c2db11f3823418192469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c813a105acddd8099c8c9281509664bbfac84ff7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c9c651ec19091c2a56fb11775dbae34f8597b2b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/cb5efe8fdf35dd4276a250e06c5577caada28312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/d042032e8d0cf199b06db1c0e77e4a086ef3b284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/d08d964b8252afdbd6e94aa2bb9949d177eec458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/d234567f223b3380324932f63ea81470101e58da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/d38b68cba5070079ac21b77244c5e1ebd3265059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/dca855220269cf4836b5ef0576676fc932c6ca36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/e1f5fdae59ec5c7f571a675ae58c53ba1bc7d681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/e22059ae5c905e05ea19cf68c63c9b0943e63259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/e899860a757736a9e240b69c8cd44bad0d706357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/e90118b5b929931e8e66142330fc8fb81b578455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/ed2273c294f8abf3b1dfd577bc11347986fef290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/f1cdae317658a2a10f4a4cc972cffc244f6aceaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/f1dba3544b954e8355e8d9be9af7d82f65acebf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/f513e5e478b5ecd33a37b1b7690544c3a940e025.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/f7d79b91d5c0c8f4cf1fac98ccdbae48130e1f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/f96075d032101100d1a1fac6ac6554e8d96b9a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/fd22b5ced5501e980b6a9fb69361b66f7301bb91.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2015-27/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2015-27/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2015-27/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2016-07.yaml b/configs/cccc/dedupe-url/CC-MAIN-2016-07.yaml new file mode 100644 index 00000000..f708ea05 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2016-07.yaml @@ -0,0 +1,370 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/05f49aae3ae4206afe0193e7c6c1f5eb5d112733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/064081efd7670ca6d8270fa56b89e1e8205b1eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/06902ebde9305237e3e39803a03ae5ed0e9473d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0a12ece6f5e89b18d99821002d65ebc3918bb060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0ca988a9f200a80b9f79052e757f94a5c8affe17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0d53b9b2e28fff9b958e519db1643ff398340a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0e45bed3bd845cadf205294b8fe4f517fcc439ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0f78c09c93ad9177b48be731c813444861350f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0fdd56c8446490505aa54baa27b30141bb26b330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/123fe99c70788cce3f2d1eb0cdbad3fd1772f358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/139bf949e20308e3919c98e6fbf52aa5291bac8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145519.33/warc/0c9cfbfeed95b91d89d28ec710c497cc982a0be3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145519.33/warc/27ff9c5087371a0de8af11ed0fbf1ee898a41cc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145519.33/warc/5c847d023653cda6213283e1ec6c02dca54ead7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145578.23/warc/633503d0ffd99c2547a2b5293d4779fc800329f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145578.23/warc/d042b5b81d40b410d2205b604226b5c2114ec524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145751.1/warc/57c66e3da7e726da1fd2f7bba7683afb1eac5442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145751.1/warc/c01936760af04e8085fbce26cd2f11afb5b4653d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145751.1/warc/d127d897f951f7c0ffcfbb9cce611fe8633ea25f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146196.88/warc/357d695b297338ebfc82a886d0f4c9ac2a6d3ce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146196.88/warc/b0d7072067f1b3e85ae2966b55fd7532cff2fa1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146241.46/warc/29848f41b9b59880e62159baf69df8ac60e5bb26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146241.46/warc/a264bfbec76aaccb2902bd5ec0871ba8498a96d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146241.46/warc/f3f0f89f6d0d2e07d047b65ad80245f7deadfbe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146302.25/warc/42a329265bbf2a5390700444702e791d9d5fd4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146302.25/warc/56f2dd36ca87c018faf203c709a59da731f2e72d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146550.16/warc/53a3cc5e4bc8d5da21638ec6a47ef0bf5388cee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146550.16/warc/bfeb97340fa34ae0468ce6e3c1246c4568049956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146550.16/warc/d4d4dba51a04c3679e0c7ad2ebacc3df79b59630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146600.56/warc/131ea725c91fe38c912da4019773cfc139e2896d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146600.56/warc/c4a73c808bfeac512e3ac21442934a3fbd74a7d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701147492.21/warc/144da67a06f91c03484fb366b4584f79e12c7bf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701147492.21/warc/59ab1754314cbda26a96b6876c3ed9378d1d5ee5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701147492.21/warc/799f7b38e376f491559f163f28eedba73e4a228d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701147841.50/warc/3e5446a4b271c2810cb30da4881de9d3472faba1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701147841.50/warc/a8a593617a9b121290cd164260313eec198f56d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148402.62/warc/10109f2eb5d6ddbd6b4063c0ef35c9954d803840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148402.62/warc/1353a097034ec0bbf4836414542bf1681296399f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148402.62/warc/4d8e597393d4101f607923541e9bc63547bde01e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148428.26/warc/69559f775ec05d72c4ed2b1903237cc0b0e1222f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148428.26/warc/e39e746e63a791bd1287bea5c0050fef04505404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148475.34/warc/0ec4af80b9f6af7eedcce46fb06b301f9688456f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148475.34/warc/65f1ff5a5a18edee949e2f8a1963b5acfcf8797a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148475.34/warc/b6378e81d53e168738157c90ff81324f97fc58d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148558.5/warc/39ed579a3304987913b605cfc81b4d41e83e929b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148558.5/warc/42af8723635852ec616208cac97a97a8b6b8868e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148758.73/warc/24ec3dc23dc6d014a4e76d15ca1d9ab25a2d14cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148758.73/warc/73a55a764efedb25d965a3183e7a862b1b825c12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148758.73/warc/e6fa67974f0c5d5b8f7926a0fe520e6360d3f362.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148834.81/warc/b23545dac38b6a44f57438ec66983e0e2a61607a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148834.81/warc/f9a9a56f5f5468d7524e51da7ec139923d8b6d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701149377.17/warc/4140fec128d1e64f51b7920a89648c6f54220d98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701149377.17/warc/9cfe29b82a3de7b0ae7532f4098fb341d7793328.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701149377.17/warc/db06395c100fdbd5056930418492a769e5b266da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701149548.13/warc/6e0ddcab872a4fa9319e38005f57e11d7b8db55c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701149548.13/warc/7bc4ca9ce76bcd396bc9b6e845440b9bbe158ef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701150206.8/warc/85d4891855910f6507be61c2648f1c24bbcdd65a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701150206.8/warc/8ecc319eb05f137b5b661e66fe75fd4202e18167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701150206.8/warc/e88ee20c03595bd20760a71251fe57a6459e7394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701151789.24/warc/bfce9b1db8d254089c356b28c6fdad24a1ee5473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701151789.24/warc/d6f8e3249a6fc27164dd164f78e6cedeef2c6d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701151880.99/warc/0ff1de7416b8303d85347461fe1bd301732a77dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701151880.99/warc/445665529b676edb82b4a6a263c5126cc82b2e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701151880.99/warc/eeb3fb5d90b3eaa81d70627d4ec91ebe6f06bc6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152097.59/warc/b228920c7747f144e788be56e0f73969f4066d9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152097.59/warc/d21d8ee065bd6d94b6ac1519781bcf9e14f5dd2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152130.53/warc/34e69ae66548d5386a18bc8c49c0a5ee14d8f37c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152130.53/warc/3b91c64195a0f935184071566b6e3257bd2571e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152130.53/warc/c778df90ec7f66da3a917de0ee8a87ac03c97674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152959.66/warc/1a9f42690d598877acd59889cfb6541e5b2ffa46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152959.66/warc/87e0ba77de9912a5fcdee24b38e6c5c89883589c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152982.47/warc/10b389af81831800da60c12bf71e0d76a736093b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152982.47/warc/63c964ff103781a0837ee2d1bd29ae50b8fc5d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152982.47/warc/ce701c15e63d063c2e0b302b47303f11e178fdb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152987.97/warc/464919e1f19e10054ae265b4553754466c9c0375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152987.97/warc/b63b0282d02584e0f80e3850efdd5d1d93b89326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153323.32/warc/0a1ac9b23607d9f60998f9fa9a5f8dff7ff71d75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153323.32/warc/7b66ad89003f9499fcdeee2b16e1737f3ef21c84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153323.32/warc/bf1bcea9c0117a9906145c4cac6c2f09d67e9e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153585.76/warc/a5cf0110bf5763a6e8d75e08c801b16e85623995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153585.76/warc/fbccfe5f56f309aa63bb981ee919940cc1b97201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153736.68/warc/27b5b9add706f2d7d3fefaf80a8dc3ef2f53870d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153736.68/warc/4590de27ba83b071bf7da12c3d2c4304bd9c8922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153736.68/warc/d1f1a71ab7bd97b522b1504bd0cb35bce7c025af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153998.27/warc/01af416b5606aba1fc73b5e8df71fc621f684233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153998.27/warc/c9a9289e6f23b0fdc95075ae5dc378f9b5329705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701154221.36/warc/2df10fe2c8456f7439895b1491e9772061440436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701154221.36/warc/c7a1ae0a3e78fec4943cce4f491f5d3f929f7f52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701154221.36/warc/f4e479411b644773ab18f6a86855b70deb61f30e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701154682.35/warc/3957e1fb1be868585cb8817a62a5a4e797b0df53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701154682.35/warc/497139004549e080e359313a5f5309b6d7f08698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701155060.45/warc/377fade5009381081745dfd3ddde2d30e8de9b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701155060.45/warc/590a58927f484a339b306bfb992f1ab91df27602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701155060.45/warc/f6fdd936f64229edf4931ecc8026c2d60095dc0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156448.92/warc/7d1441922d0b41e28b0c7de15bfbc7cfbf1cb1fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156448.92/warc/81c4295ef3523c3628c0ea317f2e2f7296bd69ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156520.89/warc/150c35efaa115a4936878ce60b4aeb176ae13b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156520.89/warc/a6e401cd0ab07c63319550413685ac69320dad42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156520.89/warc/bf20a6a8a8beff611a382eb57296c210b468767b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156627.12/warc/be50165ce532689b1d1ebd3421f1ff259b53dc59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156627.12/warc/c7c663cf6bca99e5ca236220d5742a631408cb9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157012.30/warc/3aff5100f3a4b2957211fbf88873212e93479c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157012.30/warc/bbf65610cecba12d73350955df59f4a5258f8c67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157012.30/warc/c3db3832ec071fc55b85e5663c43114c8d7ca7df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157075.54/warc/1e32471d807f36ab5576adefdf6f7e0011b36962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157075.54/warc/2b7790370653c76607dd5fa6f42ddb10e6c6a60f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157212.22/warc/31c078cd4838af3a26b788145ae19850902ce0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157212.22/warc/7b355b7f93638db673d08fe141c89e9de8ce8f5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157212.22/warc/9dbc7e67969da51e48fb23badaeb3e5934815f96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157262.85/warc/2a0bf4510d1485be57acd6cf1b1638cb58d1410c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157262.85/warc/cb4b9bebbcb3ef28da5fd4bd1113509baf9cda9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157443.43/warc/0605a180dbd184347277840cf845546f34c3728b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157443.43/warc/09875d18fedbeaae76777c3ade7a60927cf496a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157443.43/warc/1aa9eaa29419b8c99b1bad0b33d37f6d014c5f20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157472.18/warc/681de7e2cd20f83892f785d31ac287d19124f295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157472.18/warc/c76c670959ddd01c7c0809e410926bbb707ec908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158130.6/warc/2f5291421c3917766a17b08b93c42a1f47344906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158130.6/warc/717b2f04aa5c34236ca3a111d25f86993829304c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158130.6/warc/ca4ea0136a08bb95215ff12525ab87e8df4b53d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158481.37/warc/429aca4705b01187036965d379cf0555d2afb5c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158481.37/warc/d84d50910db0e616f9a638366b0770f11bc22f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158601.61/warc/274fce17f77e82a90914f1f06ec5cc29b528147b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158601.61/warc/d26f63891a0f5eea62880e5c680d5aa54ebf5b89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158601.61/warc/d3049daa2d097fa3a705a2077cebc072b07f1f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158609.98/warc/1a793c4a332f1b4befd9be26013546b273f8497d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158609.98/warc/1b66c07a9b954413da2123276db5f57cf0ec0df9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158811.82/warc/9e5a46486335d23faf6e2dcdc9858a46f9ae1cb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158811.82/warc/c1d603aeed443d6605ad8cb64517440ed2f4ba8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158811.82/warc/eed3d0a41f131194c39ee2babccaf41f284a01d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159031.19/warc/4e249965ed1e1c9da9b72f54326e7b900bad2815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159031.19/warc/b609b5aaedce961a351bb77f6f6f94c6b48ac4f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159155.63/warc/467ebfe115fe30fbc7cdf7aedede9acb1c61cb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159155.63/warc/4e4465fd5a38bedf4e76e4896d31a5eaa80d9447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159155.63/warc/84d2aee06eca22ebf97b55e2191e228248b7a9d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159376.39/warc/6f87e9ac3a7139311eccc22cfa5372540fae3c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159376.39/warc/bd6b27f6931bb6210d8277e5c34cbd7fa2a40266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159654.65/warc/a2636da01616caa2b5c48bdd224cc8391f4b1004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159654.65/warc/b5c82e05e159f0cb268c090bf5938f0823dea506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159985.29/warc/35e10050fdd597addef98a69e8f1825e498ac212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159985.29/warc/a91810e331505c97d3fdfcf6f2eedb05ce3dba59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159985.29/warc/ef0fcc27e310fc82f55e5ef4425d3abbee188ef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160582.7/warc/7c59bb6a906042823d5ed498b3c8c03623d6539a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160582.7/warc/e7234f2e1ec670119de157fff602500b9eadf5a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160822.87/warc/0b0c0a66ac679a799cc95d439da32595ddeea6f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160822.87/warc/627e6799bcc43f8a6335f820ddc0e5ee1e7cadc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160822.87/warc/d7d2b877d0c3062f143d7f4b7534e309a55cbd93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160918.28/warc/4cf863c9ef619b360f94907c91189bef53711a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160918.28/warc/cdfd1ac3cfb141d11340f4758eb7888ffd358a45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160950.71/warc/3438dff1d75e00fe4402fc7e4aa66b515bf158d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160950.71/warc/51b195ad1f84aeceb51abd7b91768d630981ff9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160950.71/warc/6b5ba1677c5e4ee9566b1569f83f7c2cb7981790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160958.15/warc/1abd6f9e295ee8e77650fcdcafae2f2b856b4a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160958.15/warc/7b39080694c441bccf1d440d3e7a47a144fae3f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161718.0/warc/539e0936929b79549f34b9fad1925a853f940dfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161718.0/warc/bd4a0a8336f053aaf924534dc36fa091a9da4dd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161718.0/warc/ed521373fb3378ace9bd22269c36f990ca2e7850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161775.86/warc/c2bd0380a0b22060630a4de76c20d7fa19c145e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161775.86/warc/ceb12668a64a0aa212b784c313f34cb6611285b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161942.67/warc/039182aa90fde08bea61da1ae1b964fcf90acbf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161942.67/warc/2d7c6dd318d83d40e393b88e8344aad41703fb82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161942.67/warc/3f3d9c30dd1595741a9812531078585fc6f12bf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161946.96/warc/5badc4026d2375520e7ef30cbaf0cbd645ef4dc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161946.96/warc/822baf6786b0feb152f7e3aa79c943a16c534091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162035.80/warc/525e9612b0820efb86f98065722ec8c99da5914a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162035.80/warc/a99fad18e2df55410959db7fe45742a04ee0dad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162035.80/warc/d526113fea936bc6f8f2d92bebe72ae4de357c9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162094.74/warc/abc6b9fd84be82ac57a6fed81bb0dda660d5d000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162094.74/warc/dac16dfe234065e6c58a2324cc22cebc9a7d0c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162648.4/warc/1ec7aa08314d44bfefe3ee107784f4c71effdc4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162648.4/warc/21c0fc1a5702c9657fc496b5684e7fa2cabb1332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162648.4/warc/4baf532edc230a069532ab8beddbd061803cd895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162808.51/warc/596a911beca1e2ddfe9ce32df17faf856b811a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162808.51/warc/efb9ab9566c29dbace2a8323ff731eccbb10c7aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162903.38/warc/0d21b2401fe43fe77e10f0480decabaa586ebc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162903.38/warc/b0514761a8dcb611ad9bc6c5262529b850624fc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162903.38/warc/dea51a2a05990c471c122435768e59902c295cc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162938.42/warc/8a0d560e617cea0cb9463ead6908b90347498f52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162938.42/warc/e75885e1bfc02871a63ee9691fbaaa40ae74d7b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163421.31/warc/1050c6ef24976d411223caa0b6d47c6761a4b8a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163421.31/warc/45dc4d681f664458f3990447184009a821c27392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163421.31/warc/e783d5a941f5409adbe7fa184cf2cfab91a96614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163438.83/warc/5a8113f236889d20067e446ee545ba3209f8bb9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163438.83/warc/cd5acabf28f20b9b01e27fe2650916525311a757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163512.72/warc/4bec65625c7d5498cf0b37cbd4b22722555f4c36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163512.72/warc/797db8be4ea29c1706142e5df339634a0e47b239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163512.72/warc/c25d96cee30e13553713e53992ee98fa78665970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163663.52/warc/473d47411333157c0873de9b5f357d2e367939ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163663.52/warc/62fae6ad0990f3155fa4cb1d085e46999c21f1da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163729.14/warc/1d7e2669b681f0edfce4cda3dc1922b7cb14f68a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163729.14/warc/72386b0049ed4bfbe9517b3432fb01891e702dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163729.14/warc/adc30da79a737d554211e9ac7bf8b97ada3a8886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701164268.69/warc/c32d006f03430aaca6ccb0d94919035df95f26c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701164268.69/warc/d2b92d6836ef6217717ef85057c53284fd4c0011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701164289.84/warc/08fa41142f80f9c37c0c957d5299847821a9b862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701164289.84/warc/7067621889cb387c3a4b25515a0d2efb9b197abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701164289.84/warc/cdead2e1af1efeb8cdbf8823bf353a3547d181e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165070.40/warc/4d42140548f2a2264cfa51c56410c8586819dd98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165070.40/warc/d099931fb551bfc51341e3fa8700900645b082db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165302.57/warc/15d79efd894c802ada3d454b51cb57fff3a17fc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165302.57/warc/63650a69f9f03fa42cbec8cada9eb4e11f6d349e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165302.57/warc/a32ee402832cf4e94df89d00b586ccb7452f9e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165378.58/warc/56b1e41888abf882e3cd4212c70000578fab4a1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165378.58/warc/7751bb8b84893aa0203858391c4ee8065eb28e94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165484.60/warc/0eeedf26a6ee1eed8157cafa9227a26b76747c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165484.60/warc/a11ad7858b199fb84f9eeb6208d4891b1f0e280a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165484.60/warc/e71b4712e7068c05f7b053256bcfef35ee62c3cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165697.9/warc/37edcdb3bb1970699f18d4e190141535f4b9d1ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165697.9/warc/d9e4da13c73cb2bcd48e718b559619cb9279829b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166141.55/warc/13316379c80a28b548668bcb59afbeda703deceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166141.55/warc/6e153decdaa7aa505c962fc5bf3fc1081912007b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166141.55/warc/b48a555cba0f4d8cbd65fe9c651c0e0df0224632.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166222.10/warc/51325c2f7c0d1d082412ee2de81d7de46b9f0318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166222.10/warc/fa223e1a33a3111bee08ce3f062c5adb3601ebe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166261.11/warc/6f99e3d44a3cb64195d795f4c91f2c384ae45303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166261.11/warc/dc953bb4c19c86ae35093de3b356209988f3fdba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166261.11/warc/ff8205af948e70ab27d6f4aa689e355e54c93a66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166570.91/warc/9cdd4a304654561408d2f078cc1f64056b796c5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166570.91/warc/cb2e8aab899ea991bd05d2e7ce85ef209f6326e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166650.78/warc/2c6835f4113768dbbdc681a4ca18eea76d964601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166650.78/warc/37bff513c1bc0b03b4b2c8a7985634850b00861f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166650.78/warc/6f20aec3a0bdd67b5ee98771ce840bae07414350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166739.77/warc/34f5924ea4fecccee99c5469d7facd1afd0031be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166739.77/warc/f140382d06a5ed387162a3da5e1eaaccf9f8d9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701167113.3/warc/06f5f4265d87c0d9de0992090b948e8a03ad9b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701167113.3/warc/09dbf45cae5c25b0812544acc87bb8c92ad3a6b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701167113.3/warc/e35b0fb0f89627c2ce257e228eb3ed08f7677960.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701167599.48/warc/082a3bebbcfbfa0baa96319e7c1774aa87b0d81f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701167599.48/warc/86186d2c58e8954aafa81b1a5dd07e0545c44c14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168011.94/warc/373d4f4286ac5fb187a12e3cf6e01449d66b8c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168011.94/warc/c982d3778ff8c655080096d0615c3a279668e663.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168011.94/warc/e041d10d4fc20aaf54edf20bf91ca55c4b1464dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168065.93/warc/3d9f5b78c57be743909eab609627f68c8aa7b74e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168065.93/warc/75a1902bcb7376680e94453cfe44e82d0e34ad8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168076.20/warc/19c8c545c8596bb53889673a9b4c0cdf49598ded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168076.20/warc/6fdf5eb14022c9521e17b0c2e94cd28315f2d72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168076.20/warc/8063ce1bb2d483729d4a23b2e3d9b943b2dc9a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168998.49/warc/53f33fb792debe7081eb53196ebe5daa76489fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168998.49/warc/aa984219440f82eabc6afbf480eaed96c972a9f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701169261.41/warc/1b4247c48de50dfd31421c4f2f0865f786efdbb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701169261.41/warc/86adec9b94638c57474c76c379799d425f020a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701169261.41/warc/8fa6453ecdf7d8ace8119b4150419a094427f7df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701171770.2/warc/1f557f7204c3f503f5eab05b78d3af9b26495b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701171770.2/warc/9b8a772a0f4619a26325a5b97a23933085165786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701174607.44/warc/65cea208ea7b8b091df680ccabf28e41f31c7c15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701174607.44/warc/664e1fda019ebd489b57c3eb09c360f3aa6fb6f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701174607.44/warc/fc6aa79272195bd96518b45a076fbc6a612089b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701962902.70/warc/8ceab798c417ae67ee41ca1b174947716c8f97f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701962902.70/warc/a81d3b2fdbcc62a3d1445d15331c67d805a68515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701987329.64/warc/86690b79f6400f2161572c26b1d203c49ffb9062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701987329.64/warc/92a4150e93c2ed886e17f8e63d7695e8f8037842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701987329.64/warc/ef0d1d52318f77e5342f15c306a0bb57df5462fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701999715.75/warc/8bb54d6e26ed0578abcef1c9a2c94d909f720c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701999715.75/warc/977686eaf4465583e5dbed6d670c6bd5ac067392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702018134.95/warc/c8368ca9ac0c33345f6d3b38e29d39bfdd33b4b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702018134.95/warc/e9ad39de83027fee0851caf04c84c47cf467a22c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702018134.95/warc/f14123727d7f7bcb26f6361c91633b0d53e268e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702032759.79/warc/50fa8414e09842890981e633972bd665d47721f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702032759.79/warc/88f8080762b4229c78ae4a6d5483c98c00f6b6b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702039825.90/warc/08b9844bdeb0fb68a5de78dc9fda0cc2252fa161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702039825.90/warc/5944b476f470114faf3b3ad740d90e94ead7c2e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702039825.90/warc/a13ffd24140c17ef6a5b31b6e8ca8cc8126da0c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/159e2a8318cda60a707f30882e70dea63e01f8f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/16e91786248c38aff1bdca8b7cbb0a045ec47b52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1a5dfe2facfa03de48baca6198081097ac591960.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1b1fbc90370b2e8e45b29e4e2b9029b190570dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1c0c1b6fcb1780654b127044398a21647345eb82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1cb7ee84a4ff94b8fe7621b116cc7eae6b58036e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/2189c43bf4994407824030905b5513c2eb177646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/23d22e256c1b48c37f73a3c5e060d944a9dd3bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/28e5b583d0e16ac8c4f66dc50d3b69740dfe57ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/2b864bec459c6f027407bbb0a7bed5281e1afa5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/2c3e7bf0551e9c8f18caa9eab15f4c1585873158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/2dffc8c12542622f9ed68addbf2dc2ebf3b1efbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/2ec086199474cbc3732d5450ec28f31b691dae9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/30aebeec480c86297015e28a9d0cb21df85f60b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/316de4722af6a278c6471248acc8248d2eaa99aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/3408840ad51e0fb1cbbcff4dc2c4e5d47e5717e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/393fc2c5aec8cf71f76ec9ed3617af3b22b8aa98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/43a0ef3c88c831a2e1b39561f5bebddcc5a60831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/45061ff3873c946fd18cbc63fc3e9c860172f604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/4a2451505a65ecfb544f07f274a541ee9026396f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/4ca07bb59944223662034c117a563cbdf6879329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/4cccd8e70896d9462aa99d84dd36beec14d80e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/4d4f1932897392927386a60eab5f395392e12bed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/4f295088584ca2590c7d93032d9959f5944b890d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/51f794e0204d40f4e1298163dc2754fb8d45db40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/52eb30c87994d5dc857098377631da52c6daa0df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/56452c88a222ff11c58cf8c03b5a3e8d62dd052c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/57edcedc7a5a1c24c40e6aa17e1b69a1c9e6f246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/608e8f896c926c9dbcb8a65cf8aae8468ed271d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/63bd0f1f25d678b3b3e28977bf922dcdc7f88033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/66288724e0e4b5c0c6cd1971fe1172e43bb45a3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/7032b271bd45285a057ebbf0349b8dd2eb90fe5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/71138b1cf457d9ad9e418a271a77842d9b475702.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/729e7b8c1e96e710752d4f43c2cba80432c10235.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/746a192f9a4ad8a12339b8540a54e41641ab6126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/77569ef69b27adfa5943100b5d5adcbe71373eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/7765b99c7130c61f8dcb8e0ddd73a74c10d8e07e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/77e1f29fc193f91fba83a5af53c1f4c53b610b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/7becb8adc5a96d516dbee44f4ca157b9d0040139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/7fc36465fac0124bc90bf98aa2d93beef581f59b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/80218e35d2e91aab41554927bf8356b4c1f0495a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/81f41eda906657a0004573a4d926da6cd52edb46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/8e6125270f0186bce58f765b3864952952d73801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/906b91706e18da2d4b6d0c440e4745b09e8770f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/918bcd083f6b038bc630f1f4bb4ef9378b7c3ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/95fe23ae49caf3bd01b07c30733f436c671381a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/96939a91f09791f429f8bef75f801f9e006e0ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/9853d6f53367eab0564198b56b50d74604958a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/9bd38740e4616bde8b6eaab8666405518a9133b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/a32f94e1fb9e13cb12341141a6a73d7aed4139ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/a369929c64ddc35c5359f9feffc14c5049ccf0e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/a45146322330d6a4b1cb2e1595a397bc4aeb454e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/a65cd38caffea392a4a653acdbbe4c2b2d3bff03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ab1dfd683c5b5ce067fffde44cc889ede3529b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ab4384863c8223a0a3b38bbe1e42f28139eee2f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ab529ab455f2cf9cd4d5e03a4bb1f724b0a9899c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ab91bba7e34e74b98d1efe5b81b89fd7ed4eae57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/b2b7a0247a99abb720a6a5a20b15edfb511a4919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/b50bf1020d699d4910d884b9d16fd6db0eb293de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/b6499c548a11e0ce70faed1786639f04982a1dde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/b71ea075b2e194494a97b372a8b5b5cebab0f92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/bb0d087ffcfd10bb9b07ab7a8ffcfb00d8fd5c9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/bbc08f3a5bf38b99116d1333f71ebfe39023a0ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/bf8cc9e21b16b5a06815cd1ab37e9c81de214f13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/c0ba8f00d2ce6633a93ae778926bb58831c85dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/c663513b0c0be4fe80f7b13ef6a232ac4dcf4e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/c76d80cfad7e513c34a1ebc4b3debc135749daff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/cc250c7bc41f3edd2e6a068c69849de2b802af60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ce4f80f1a9fad285b01184a248982b8c15a40765.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ceaa68aba086fb105f5d945d0313c956fb3ed020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d2b1d803a75f6336af72a1bf101b36cbc9a09608.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d2b4e85ad7a861d61944fd9a5cb19e8f6bd2c89f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d593ded4ef3481fac10fa845c03bfc486c2d41e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d61e0e5241509ebf1bcbaaa20ae363f6670c13d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d81c07d98b3ceac2e7688a8b4565c17b49aae813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d9174dcb5e638761b43fe8f572ed4e97fe6e2137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d941983d0aa819caa770f8f964eac938b6a9a948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/daa6c6d2f410ea748a2e014855a274ffaa2e7698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/dad2c6bcca5e7d29b72fa44f9b47f26be1b1b509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/df44b2e4848666965450e936a8f0ffa5d59a8da2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ea10339d0a7ca362a80ce2785a7b35d2a5e50d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ed623442ed4c0b9579e0ff0433c4c4b7a8f03780.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/efb24d28794ac99225f847fddfd9a3120cf81042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/f776e4d097914493e3a1fb916b48f9eb8604dbf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/f8364ca6ced44c98b2965a830e61c12ea320e95a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/fc98243c31151afaf8edd4749e8e81050c8deb55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/fcd0b756fa173ea7ee426b6ed117cce9b0f56ab9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/feb185b9405ed185fb257f19611919394226d044.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2016-07/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2016-07/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2016-07/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2016-18.yaml b/configs/cccc/dedupe-url/CC-MAIN-2016-18.yaml new file mode 100644 index 00000000..b96639f4 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2016-18.yaml @@ -0,0 +1,243 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/02ad2ed1fee224f791c9c549bc74fdd92d824be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/04fefc4f72c4b669d1ba7d4aebec224462863908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/07933d9fc10af51b0b2256183b1c6173d970c02a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/07d8569d33b639ebc3b7ed685bb2f14abf73a2df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/08f2a8fa13d3a2954c948becdd637982e213fa8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/0b0cf0d54236cc16cb0d30b39bed205cfa6f6516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/0ea165be481a26d0aaef7a55bf3fe46f9b5794bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1029998319bca8ae738420773f74755ca94a5ea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/12f3794e76218236d4c4bba5ff20d88b652edb3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/137758fef2a2a744b0214b24259a5b20a3fd1077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/138f0a132dc1c2b7d92d75dd4bc63743fafbca8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860106452.21/warc/649256234c09ee8791a63bfa7443ee74fe5d27db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860106452.21/warc/f07e8f6dc32f1ef9e6cc272fb55cf6f9a3f38129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860109830.69/warc/be467d8cf1a8f447389498b3e82af65f2e6b1947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860109993.2/warc/7dc0af9cd18fa826f05e27482f9ae182e37e3537.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860110356.23/warc/6ab8253e611d85b557c038c7ccf239bc5b214d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860110372.12/warc/863728089fce0ccd231d4e97d3b3eaff42944a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860110372.12/warc/a0451436448ff7526fbb6a54ee615e30fb5208fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860110764.59/warc/8771fbb14e98f0dbb5277e055d33e94549156a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860110805.57/warc/0178ba1e5b03c720fb27e01152e1fcbeb03df806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111313.83/warc/e23a8dcec446b9b7a5ddd6729c3ee3fee351572a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111324.43/warc/00436312003d5cd37eeb505592a47652f6841aa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111365.36/warc/4028eafbf1e35c44242f3b82749c75d70e62c7bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111365.36/warc/62560e5e08998938947358b08045285b843e6bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111374.13/warc/9ea9ba9c86e48d4d6166946433c7ecce81f5b5ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111392.88/warc/ffc23555ddc6f4b6a99808b228d0a245b92ed5d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111396.55/warc/95fbacee85f8cb7e3ca0b5467a5f1fcdb41a340c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111455.18/warc/6bf54c60dddf8114f13e9f0b0bd77d6479eea549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111455.18/warc/bbe84390f2387a27a977202455aa02aa01f02a0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111518.82/warc/2ee7f97a58ebb9bce3d875fbae50ec24af67fcde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111581.11/warc/98edf6abbef21723168f4440c8bbb921e039d429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111592.84/warc/3a428efc2a902db87b54f4041c93e55a3de6289e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111612.51/warc/68e0eee0321e2bfacaeab5c43a06d30cadf3d6f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111620.85/warc/9a569b971308f3aaa9f5052a01f45ad4bf134916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111620.85/warc/c3a6058002c35ab203a08041aaab0b950f8a9201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111809.10/warc/0375f7d95ce59ab06fb10385714308c466967976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111838.20/warc/b1b05f2ff195e795182aab0319a56ea77197d3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111865.15/warc/673e637394b23875d0fc18a040d7c12533d848b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111868.79/warc/9c40f1a45553172bed3fc23e9442d37db4e1c6f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111868.79/warc/d5d2c614140dcdcfe36423caeae76012ae5dfb56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860112228.39/warc/ecd6ae672325c27041b986ef242bc5b6bb30713a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860112231.78/warc/5266a6fbd1843bb028cce23d7621796e777a8b82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860112727.96/warc/d4327c703c0ab8932f1feb97b9ee3975dd8ba7d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860113010.58/warc/0e3d1e57d43907a51a1b3cdd88a37fbce3c5d277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860113541.87/warc/cbdf06d5f2a17b37c83b4b2adb8d8f21c1810e27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860113541.87/warc/d22090597533e91aff4c4eb387fa42df676b8a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860113553.63/warc/bec71eb14ac9e3ae0bd66a4e6ab40d0a6056ab9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860114285.32/warc/eaa925efac0318228e203a7e893d0a08760ed183.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860114285.77/warc/328e304c0dd5134fc9c98d03840fbfda008a0979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860114649.41/warc/0c6e857717677f0cea92e26a07b647866504d5ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860114649.41/warc/9464393a00955c5b27013dd4aa3275a69489aac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860115672.72/warc/d0154a3cdaec18710e58d4e4ef5484c6d69ad43f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860115836.8/warc/105ac07d397e8c94a2b5dcc7f24eb354837b8170.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116173.76/warc/f77c3062dd609499adf9dae9b80a2cef3ebe8c31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116587.6/warc/6a9eca521252000e1f5c9d13b3c3300de4ffd850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116878.73/warc/29261769804d5db8894220743fcdb0a5d8b975dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116878.73/warc/fe40f5ec3a4496a0a6e7879df0de158bef3b34e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116886.38/warc/df2d3acffcb2782f6b73ee4076f8cca7ac020f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116929.30/warc/3088446af1e073b7a8aab22c35f71d16499fb493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860117244.29/warc/8f21b904c5db624319af2112a9f0ace564c6414f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860117405.91/warc/72ed9e0237e6be98eebfdaadd99b4df2d3242747.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860117405.91/warc/b177d0d4a95248e7d75d8fcbdecb8aacfab97282.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860117783.16/warc/f7f8891ca3dabb7c1036f07833238a756dc85c1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860117914.56/warc/a60e4f28162bc18bab3d41181ad922a98f344575.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860118321.95/warc/5e79ea63e026259fac39c38931aa373a14e7ea92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860118369.35/warc/867643938b126918e9dd1c2c4bfc163b02b67b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860118790.25/warc/4815a368c94544765c9e1320519ff8fbf7f7b6be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860118790.25/warc/b2bb50d5cadc56ad302ba505eded272a6064cec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860118807.54/warc/059ca2c3307bd469a32354a49e92f0b660cfa797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121090.75/warc/636897e65abdae48db89700747e65e15efba8aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121418.67/warc/475e0840a0e776b3f4855a408b7be0c1d820fea1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121423.81/warc/7e23cbb22636ab877b99e1df41fd4202b5ee41f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121423.81/warc/d12d89ff60cd03626d0492aadb4026e07c173b1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121534.33/warc/5ef6bc1433cc4c39db6e728bad06b9b80ffd54cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121534.33/warc/9a74f55a1725f7dcf2ee4637e8245072314a88b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121561.0/warc/909c18563af208d28e7fb20fbfebf8cebb55e432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121618.46/warc/c56ea4f8bb22d49c3d5be49cb6a4dff259c368f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121737.31/warc/2c6c2cebaf409ab10bb2da32d63299c717ca6557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121776.48/warc/3f89560949636cf58f3d24f25aeaf817cb0e9162.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121776.48/warc/5401ca1468a4c2f0a7387bbd70eb0e15670151c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121976.98/warc/70cc7327317f5e831fc7ef5099530d474b4e04fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121985.80/warc/38d9c4ead892a754f6d2576768b74c7f2972207c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122268.99/warc/f851abd4463d6070c62fdc1cbdc9de455198e9ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122420.60/warc/1581ce81d197476cd87f0ddddda7eb52f4efdb1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122501.26/warc/16a8fe24c9efeae7bf7573a1928d276e8b767f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122501.26/warc/3361814ce982830724643bcef6964284572a5b2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122501.61/warc/980a8a576481baa26047ef029379bfbfc2fff53f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122533.7/warc/59483b373102628da95b6eaa88e291bea52b9de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122902.86/warc/55c669b877d5c5287b4e67f6a1098733b7de72ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123023.37/warc/134c1d198539a2c911550077f906b40399804944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123023.37/warc/73c23b4cf66861b62564d256b3f9e309855b0e0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123077.97/warc/d6d2027c2f9e2f1d9a469dff16ab07dad0e9003d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123151.14/warc/b8cd303c6d09287c8dce6aec96116f21739146a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123840.94/warc/437a49b5883a1c364d864b697b486e61fea68546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123845.65/warc/9cdb3921c2108e4d0b7a45ce690425f282010b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860124045.24/warc/7506cb800750918df8a5770ba300b1dbddd28dfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860124045.24/warc/c91fac6b543f122bb0db56be07c16d1aaff6198e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125175.9/warc/afa77243f289725a762189abf4f74cc41a4af8f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125524.70/warc/d334ff11403fbbe3eddc374402fc6fb2fcca2744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125750.3/warc/a0c5eff5dd4020850bccac9d3c20fbdf0d23b0a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125857.44/warc/5a8348ed5fb41db3d34c885fa4c801e15d49e88e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125857.44/warc/c73a75a5b8ee098637dbc4fc6626983e9f7d65ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125897.19/warc/57fe6f325a32e24257ec7d4837603bae03d1615f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125897.68/warc/3314410df254dfd3aba14ff6874ea61ecbfaeceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860126377.4/warc/bc3ce421f08d9bec835cc8ecd61477bad7045b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860126502.50/warc/19c98497772ad3937316896c8cfca45c8e385feb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127407.71/warc/044ed577652d3e23fbb607c3a75c8fd038b929a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127407.71/warc/0b2df43eaf3abe8f874370a83fb142133ed5af46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127496.74/warc/016c72085c0b6bd2ae9bb254fc36699f9e721b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127870.1/warc/bed9e13d5286a4b8642895ed1a81534edfcbffb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127878.90/warc/9f3029ec1208da918896f0e6a68b487335a8570f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127983.53/warc/8f35adf123b9b76fcf0d55b63b5d14aa3c4c7fb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127983.53/warc/9e3df30c6d2cd4414e0f3ae224b060b04ae89cf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860128071.22/warc/dac057967b388cf93f9786ec0f0efcf9b3a172e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861623301.66/warc/938eed11700dbe91d7ebf2082444d34e98f3533d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861700245.92/warc/9b53cc29dba0608d2ae7b34583786d075ec30a40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861700326.64/warc/dec4ef0832350932ad37deb4a37d8b38d3d38106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861718132.40/warc/0c2faa101e14691103f441971b38433009369900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861718132.40/warc/1ae39fffc0615abc582a2ee914fc01dbd689422b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861727712.42/warc/32cc6e6177887a93b3e4a2c05fa66c97a7264e01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861735203.5/warc/018ddac1579ebe3ae11a6f184c67f14dcd4c9fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861743914.17/warc/d1a232101dd9929fc3a7fa08f5f14a070434cb9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861754936.62/warc/5f0e0e8208f15c42337d8bbf1f712f7a1ff71fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861754936.62/warc/6fb43bb8ba0fbadef7374094271177b785674430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861812410.28/warc/c9e586046dfd6bd904052e887c9c35aba9f78056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861831994.45/warc/24e2cb778678ae151f9dff6b5fd1dc537a51df13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861848830.49/warc/980ac1b141e0e639f619c851dd4d4edd7770d4a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461862047707.47/warc/38ce08b15804b1d074b34974532afbe10a3ea8ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461862134822.89/warc/94144cee1e7ba2459eaa3ba315d1c7937eb9691b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461862134822.89/warc/b9394b6f335842146ba7f6155352182607a77ca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461863352151.52/warc/e0fc32c7721dc740f7350b08adca46b34a3fc9fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461863599979.27/warc/c1ba422294cab9573905404f4a33826c76ae28c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461864121714.34/warc/99db6148ae27328dc31776dcff42768dd31597e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461864953696.93/warc/6faa814cc5c3e5572ef30e3459d4adf277cb54ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461864953696.93/warc/af4ec0b0bfe1bd0692068f99ba3e6687331ee876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/158ef7bf790e2fb69da621ab6861e78e71727d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1684a93b2057f8bba8c4f9a7afa77bd03a10736b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1aed6a68b7a771d2e5a055c624dcdc8978c6adaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1cf43afb51ca1b0ec75d938fe0af7533c627bec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/248cb67f09f0b52ea7d4d33741e3b77db497a155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/24fb4897186d14b171f23813a383f0105cf5e943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/26e088325d420874e9f5cccad579d2f1ec362762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/27a51cbdd640c9587b07fed4a9de57f6bfed6e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/2b41fb890c8e6ba96bf39408f2b153ca8b66e8ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/32d6aa91901878c8947700d428b570702fb600d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/331305cfbb923745711244cf51de7a2c34c56d7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/363db31ecd063cddf470dfa8d1ec12af43cd72c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/3f86a1377b7e2ff521a7005441f1a11be623d91e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/401c95008bd6c529e12271f7bc73be684cc58cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/46975be847ffc1100b57b1983f36cd3033117be7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/46c459bd0b35a33be74194af79be63b68abc1623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/49788ae4a27acb20a4e3f174f4772cae2ff95bec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/4cd035f264e48c82fd07f81d74b1c8084df503d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/4dabd59b2719b09a2e8e6b3f09c5b94826e4c9ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/52c5eca5f3f796514d794c9b2e632471a93b828d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/54f084fce48093f8129e93f0536a8b980021a9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/559a67e177a35aade0a6c81876226ca97d185341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/5761a823a80c6f4a74b2f3414a7e997a0169e858.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/5a96f0efdeec604797ba7e6c68657a20c33e2e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/5c96a5e38dffed533d535ba975a2cd73a124c601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/634f0cb4e292fecc19a1d6a42b69974793ebc4a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/6638859e08cb90f28bfdfabbd5e93b9a2a659995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/677433b2012fde798406fca9231cc2fb87d33c98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/6953b6996bf8c6c03ed89f66f7463498513b780e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/6bd8291ff1dc16c4ec5b2b54fefd9c64a665a714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/6c6c7b84a0ca43853c3feac3984850f0c064a90c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/723b42e25dd82b7957e9789608ee084c9f9650b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/742972941e5145e9a7bd51f088cef53d8e7db0f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/74c1582077cc6709f2e9da265688d4b2632ad5bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/757a10b2ad3b93529a5be40548180b1069f030ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/772119f64522da486c16c4375e856a4540fb5f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/7750a85476d65a6e2364cd6870b540e2df3eff97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/777218916527168a8a1e2f4b4b82be0ce85160fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/7826c2133b2239c1c4cfc8849f49e98c10a22470.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/7c1a65251a7e819a3f231f8ed527ceb4cb683bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/7d4410687de96a9e6c09d86a186ccad279bd5e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/82f5a677616437b199f0d2595c2ad22fdcd01a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/8795dbdd059acb09036fbff8fca8d207070a90df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/8837faf405448953f5979444fcac54a4f2d25143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/8bfe06ff3c24e6362f4bfd61209ffad74a33fa02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/8eaf49f79cade5d0144d18c222035088ae2146af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/8ee7cf8ae3360626e87c05ce2980d96a644dd354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/930a7a99293ba87ad0f4c31ff8a7d2a41e0f4773.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/97a9bbc915e3fc45f95947fd84a0e95721ef9d42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/9a7e3bdda14d071d4b88382dbc00843aec121df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a08ac0617e0772eb1ab3ae3ae5a92cea4bdfaf2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a26db384b09066dbd5ee2a6d7eeedc36be46187c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a27d920af1828e04982890d8d2599b385276c2a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a4dd97fabae3d1074bfdbcd4dddeb40a12769116.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a7d2320be7cbcbbe3a9fbd9979a395975e8b152d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a80e7d7793afeda00304b2637f1b66b26336aa36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/adaaa4b882c66cd7fe42c6eca003ab431fcd1af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/b084ce48a6d3569835afafb7005039b97fd40b41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/b4ed039206b959655fad6eb8001788bc459d4429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/bb034b307b6c089a0bd59e2060ff0922657ad885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/bd95f38954abf249e62fad4d7faf410d4615a594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/bec9c7d09decd070f95ecf26489927aea819c9d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/c12c399afb55671ff0835346aa2701cf275ee683.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/c3531ca35c2be1ce6b5b4b4f205872b257e944ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/c3cf54d0d2437d61b689591932f403747f193480.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/c3f02465de71f054fb5b8b4edc4256bab0aa895a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/caf2082fc32860a944fdf22127fc08b8a8abab66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/cbfd57ea1604e91dc96fbb64ba1f55f71fbb552a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/cd4c4b999fb3e6a1c1188b3ac9a4e103ee4cef8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/cf4b55e8d4cadf89b3dcbd12f553bc8841898bdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/d15ce56fecfd07de5dcc7db3e8d9bfd08217c4ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/d5b1cbfecd89298cca305b851011079f03a8ac62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/d9b29c1ad364973c4ed82a08b6a4b995d42fde23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/e60d6e1f494082bb152f8517249e3327320b5d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/e6a0bd1b57c8cb56e2302918a183c558533d3ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/e87cb946a9038736559cdcd681fcea7e63786137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/ea1a0e2f2f999aaf037b178f46778c85401fce27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/eadc02166de92c881d6627650e9e076821dacae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/eb4e806a3ce92194acaf1ef06b00f84289f21afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/eb986c7541ebf5b8327f0508736a471c7c886634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/ec59e80a2939ff2afdcbaf6da7c3a26039c3707d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/ed9eb6686fb1e08a62ab57a8f01839b575cb4bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/ee6fced157c629891da3e7c31ac9c3afca33092c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/eec8196f4d6d3f122e7635055c04c52a06204040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/f1365ffff6a09b2f4d4f3beda95b40fb341e327e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/f1f1f5a5eb0c3f243648e0e303f419708b256d4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/f8b904d48a65e8870606bf6fe171cb186e8fec99.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2016-18/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2016-18/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2016-18/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2016-22.yaml b/configs/cccc/dedupe-url/CC-MAIN-2016-22.yaml new file mode 100644 index 00000000..d237719c --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2016-22.yaml @@ -0,0 +1,266 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/023ea1f78ccf5a1ea55ebd2a50a3aea042770ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/055d785616aa5dc26a6cae954940482226103bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/05962fcd5c51e669cfe27aa1bdea8341d0d3f4b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/0814fd8f8a2d386dbb2ed77786c1da8217e07ad8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/08726f42446cf23cfc9dd941835e21eba9f8005a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/0b9fe34962908cb8a10e50b84d2d621d1ac67a16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/10f70db88b3361d416b08781eea30d677ee6b7f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/12141e79a4c94a569b730430de9bf6e8065be3b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/142dc6b43957d103a28cd48bef6fce63d4e8dfd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/144f1babd21818c456f3c150591c759d4607a07e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270134.8/warc/0fb5b4a622f2c29c846b8abceea4cefe30c539c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270134.8/warc/3ac42c4c01cf52788b917d6f49cf443e2ed3c083.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270513.22/warc/0efea004653150f3a705b11e207f38b3606152d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270527.3/warc/516aa8952c8feac97c38d745c512ba967617e61c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270527.3/warc/cb3637f6b39ba96c1d6b83b1e27461148869689e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270555.40/warc/9834ad27cd29e9b77b664a0de43388bf54d5ed50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270798.25/warc/0265e4f0be8c189e4387eaf890636b08d763f3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270798.25/warc/62816630b2367f9396edc1e957aa6b18510bfbde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049272349.32/warc/aa8b012de8866c68de2242bb815f718d391bbf0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049272823.52/warc/8e178fe613a483fdef641f3f1252996af500ca3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049272823.52/warc/9cde0958002c5029e5b527707a209b33c55ddc26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049273643.15/warc/b318d8c230b6330946d4d405d1991685f8e732c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049273667.68/warc/43476102e3a323ff817988bd5218e2c9f82b3db8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049273667.68/warc/e09347c89d45eb8e5311efef8ff3d54ad1236a40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049273946.43/warc/57a2cace31b8d7ed5458d251e9cb61c11c4a38b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274059.91/warc/9fc8181f4b3594648c68dab2bca8ac13a214c602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274119.11/warc/61787afc0288ea7fe5a300c463a4487b0431a877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274119.11/warc/a1ae468b1f6f6b0398f05f56cd798b361e0ffc39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274119.75/warc/d07fabcd0068bfe67392b4d5549326b146332bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274191.57/warc/1d01ab15e3f94f6955c49f0294a8c710ceaced15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274191.57/warc/74aaee7cb23b09dacf86b4319364f15cb4d6a5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274324.89/warc/b6f1914763c6fe4070e589e7d3846a351f31cfc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274756.45/warc/19c6bbfa5c3488ae53481602bc3f2baeb0447c5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274756.45/warc/e35c9720b6e71c8f7bc9896445397161d960725b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274985.2/warc/0269eb121a01abc805577757402fd03bc201c081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274994.48/warc/4c852e44747891dac758d804fcf05f40eac3a10a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274994.48/warc/99d8966795f76dc634a3b59d2c1ffc9bc2252a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275181.24/warc/e5c577afde9ffb447122dc939b8156c24139a91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275328.0/warc/d9d9fbdab3dd27b54d14501dcf48bfd87fbb942a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275328.0/warc/e3ecb20e231f5c05620e9097066ecb71290b986b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275328.63/warc/9077ca9868d245e0c4df6eb7b5ab6f22fd13407f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275328.63/warc/cd6c34bfaea11164a222b72d55a96de103c5f74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275412.55/warc/a9b19de334198d437ea50402409d3efe7f15d1c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275429.29/warc/d594d52c935755abc35f6271475e97189b158a2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275429.29/warc/e5a8c5a3e23d7dd4bbf3da70eac8b623273df70a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275437.19/warc/39224df12198eacb69e69a7c3476c4851eaf8c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275645.9/warc/14e87e50d7f82532f62ec44e567bb14af84135e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275645.9/warc/7c9fe30058934d6ea65d0f244717e027d41651e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275764.85/warc/8b11f35229fc2e111db197babe4734231e5d1087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275764.90/warc/9c62620a99fa2deff9af2fc427aac3116d66a9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275764.90/warc/ff659def4ab6bdd8fb093def4007d4f3ee537b8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275835.98/warc/fbfdd3b80876a5082ae5f87bf8dfb15b05e00b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275836.20/warc/683ca3d397a77eebc0f04539c88e43451f2b41a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275836.20/warc/8b56a7dc1f8a1114c5cf19918f29f291054c7a2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275981.56/warc/b7548128fade09e691252e217aa3038996e1ce0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276131.38/warc/78d390fb8905b438e598cd97d973f6a270d47c9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276131.97/warc/4cb47542a588aa1e32cb9377e1e037beb08ce5c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276131.97/warc/c19de4ef92daf6f96c61dcef7365a2183efcabcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276304.88/warc/abe6fdf469d77b83e368951dc82c0f42a1860b7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276305.39/warc/25088c435875d7f54b7eb682d28d38b5e20c9433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276305.39/warc/c4323f50771819323a273b60fcefa2602594aeba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276415.60/warc/e7604ac6da1a7b82d8c0f2e9686add5499667b55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276416.16/warc/60e17c11e88d29b41b3d7d420742f5a6c5597aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276416.16/warc/ccaf084a2f41d7d9f33e75192c958669ee9a91ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276537.37/warc/3cd46bb9efeebd32855287f62d3041d26a6e8b81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276543.81/warc/848a619fb042ab5b52f8cabda18a8e07c7cdbab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276543.81/warc/a88e7510542a296723cff09b8f66175ba14cda97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276564.72/warc/58d6a7520ed6b55afa987ded435882c87581133e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276567.28/warc/2d153999ffa3bb774e599d52100165e2c134fabb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276567.28/warc/2db139912cec240345014ae849855bd5faa7b734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276759.73/warc/b3b06360873d02cba3010c16d1e0510333d1cb34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276759.73/warc/d0f6282efb749f6f69b47aae98d9ab383491200f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276780.5/warc/ba10909ef4f66091c4d1a4ddfce74c4e6885b19b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276964.14/warc/66c251bf3c864e7cfe7b5a48ecf0eea5b79276d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276964.14/warc/f2c5ec9f2edf18aabdb055861fcc17079f08ba26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276964.77/warc/2ca7ebee667818a2698128b84829a884a3a7f394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277091.27/warc/50a09138b891caafe6a22164c242374521dadaf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277091.27/warc/d46538887279a170eec39ac60d68cc6154631a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277091.36/warc/8a1f5b442df06210d7978ccc8cf03b100fd117d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277286.54/warc/4ef9b1ddfcc7a7e67f42fe2cc18c313a663a13a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277286.54/warc/ae9b527b7653f8d73286a84111be7bb943f515c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277286.69/warc/fae85c602b92638a906aa681571e6bda8f0f35df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277313.35/warc/3a19209da26a06b10f4eedf21d87f8c62fde823a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277313.35/warc/775325fa306173e3667255372553d85c6586e7dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277313.92/warc/80c5fefbd0ef7a5f34ee415e0589c7f0808e9270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277475.33/warc/8ea5f45cd0a7b164c91c817e2f3ce9a08df2060c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277592.42/warc/0c0eda7f745ac7ca96ce41915832f31b337c387f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277592.42/warc/f25d8bc4dfa88bb8c77c50c06dd43652e4fa7791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277592.65/warc/cd16c124fd9bb881021c55ca8717abe3cca54ff7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277807.82/warc/2b590016f1b453a4843bc3debb77636da2171736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277807.82/warc/40fdad64fdcd11ac4fd9758f89294a3398c7109a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278042.30/warc/92ca1d7d793ba4cbff51bb7207483e8146266038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278042.87/warc/3373322d013efbeb5c6e2e4703f1fd443cb402c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278042.87/warc/895bc5eb7a0641e164f3f1f28b6fa11506dd9916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278091.17/warc/d395e750b0ce00fcb6220b8d6c5f4cb881c8d715.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278244.51/warc/126614cb87083b31ea7336ab42c0d291ecebe895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278244.51/warc/4ffe9a6bd01814470bfb5ede010f8a350792037d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278244.7/warc/9675e7ca2cd9cdfe841725ace9e43a5d57eb8f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278385.1/warc/03b915e88dffec74182889aa59a9c376be4341e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278385.1/warc/a05f9958fd181adb185b510308fb705a1fbdf343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278385.58/warc/428267b8ab6b8b2cdbdce8813d1243b6d8c10877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278385.58/warc/483683ae971352e42ccafa5dc7800bda4bb69676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278389.62/warc/204ce28839dbf849338ffe3756b351cef82b5ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278417.79/warc/347be713079ca852525b610be75817046b64f13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278417.79/warc/fc5a08bd50ecaf7699ade2532678dd80283e6e52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278887.13/warc/f1a07e5148e8ca1afdea9c510c49ffaad644f2f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278887.83/warc/79b08bd4b14d89a9cb800cfdc94e75817f913bd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278887.83/warc/a5cd9275ca71883d2e314f2dcb5f491f0948b69b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049279525.46/warc/7599bf410985ff74961c028bd2b9a95e235a9c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281363.50/warc/001a484d97f5aec260e785c4baa029b218d8fff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281363.50/warc/3f51f9a3779e7667d738b64438415bfb8a3b54d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281869.96/warc/6f42a03ac598c0d82142c7943c1865e5ddb61f50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281876.4/warc/0298570560c6ff303bbd8bd0244220bdd0c7a7e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281876.4/warc/c7a2eebe43b54eb59499a00f8b254e35440f09ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281978.84/warc/d9217206be366a59dc1944fc4413b46f596f0106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049282275.31/warc/679680f8e9f49461fb81900161dd23a6b3339a0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049282327.67/warc/180b0e5c1ddaecba1e90671b342397919d2b4700.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049282327.67/warc/c6729985e5a3e926ad773df50b911128cf82d8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049288709.26/warc/3f51e3bd6da449bb6d46a54e11f843be27eda6f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049288709.66/warc/afbb8d9756283db9889c9e3e6c0acc52371af048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049288709.66/warc/e6c0a4d071efb4d237a7714d56d11a074397654a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464050919950.49/warc/a3632fb56660935f1524ef9883ecb525c41b0983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464050955095.64/warc/8336391db7fd05df99ee9bc9c47001e3d1acae5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464050955095.64/warc/d9d0b414a1f02000e904d3787909bbf4a24abf2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464050960463.61/warc/dd7ad3b4f041291df52495d8a1c31f54eb02e660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051002346.10/warc/0b68a26193c26b3ad41edd4b514738aa33d77693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051002346.10/warc/ee0898f4a0e84521a899ae6319ad942db107462a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051035374.76/warc/9ee7dbd40264a53adbd2416f4e631257a542d3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051036499.80/warc/330655527f813bc6a3e519e5c4d9d05cdcf7542f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051036499.80/warc/c2ea0fe150acfa577643fd098fa5efe707061477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051054181.34/warc/832c63eada2d15433c3519e6c269331494b10905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051054181.34/warc/c803780b30c10c1188154d6179b8ddad4de56305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051113990.53/warc/37e738546884dc98b4bd219a735e75026ebb8d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051114647.95/warc/05b2a5df768e38b4bcef643c8f9d2131e9d48c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051114647.95/warc/d773a47e425637f79a866ebd92143ec26273aa1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051151584.71/warc/fa1d403d588d40c308c0ba0f1b388488cf9ae890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051165777.47/warc/9913c77e6184aade7f5c77e0747922b00b7c5018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051165777.47/warc/e639a5fd14dded69f8949ce088f54c1f6c7a769a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051165777.94/warc/5eede89118cb66959a62025c16a5047a35be3943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051177779.21/warc/4d5f39fa2dff26dcf4c24b175bcc8d279ff74353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051177779.21/warc/d527a95878eea1cbaf93497be70bff1884893836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051196108.86/warc/91ba6ce91790035a29fe3ce0980c9a333f8f4d3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051268601.70/warc/063914b63c4af790aa7ae75b97f6e75b7f512eda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051268601.70/warc/23f91cc5d98b662f3583c36d4b13b77baadc0368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051299749.12/warc/f77126f967bde05e49dd822a12bc9684d3259818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051342447.93/warc/5985af3d2e4d0c77f8018cc8cb8c0bd81314aed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051417337.23/warc/1d11e8f87c7b525c34140d00470f61b1a2c83dd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051417337.23/warc/6572f69956f66a3acf9ac3bd187850fca06fa89c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464052868853.18/warc/2144eee96b1dab82c2fa1d2d1a6aa1b68123b0b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464052946109.59/warc/78585c4311d326f97aaa5e04487643a940e78df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464052946109.59/warc/dd5cba4c93b996f7bd1f2f3cbccc9cfbc4f051b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464053209501.44/warc/0ba24a7f2e4caadb01001906cb31c61a344a3818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464053252010.41/warc/222806a656b6396ac05e3dcc446763e080bf52ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464053252010.41/warc/6fb19637cfe613f51fc9b98912b2113e81e34b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464053379198.78/warc/97825dae809018aa11326963fa277d43a6ff84e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464054526288.74/warc/b4f3cc7be6d4496aae30f6b591b37fec7074f4ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464054526288.74/warc/cbef754985736177f34bc5119d9818ad5cc3fa10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464054915149.6/warc/09c786d1ac3345c7cd90096c8a9cc61a49efe6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464056639771.99/warc/0d850a5578e7707567ecabac998903f1fe36d692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464056639771.99/warc/6bf6c192177fb23ace50b3c11d46ca1e50aa7e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/17b0e7f3dd84583a2d2fa0e9a47f5b04a60f4463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1974fcb6d4710a6f35a6f12e03c8444778cf1e93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1eb5205bc7dd9a6efad042d6f01310218285b4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1f7828ebda49f82c375b12f3442f1ec1f2686807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/211a85f6c0469820aaae9373982fe9333a57b3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2248a0cd43745c82c8a220af7c563d1ee44961fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/22c28bf13d10734a80c9621e729be48341542bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/230e88e80858dd95d62a33e441f49de9442e2b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/233eebe1c775e80369632afbde1ee26e8e848455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2399532c26fb31020a71455b15cb54902e5e4d32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/250d520ab0c3f07ae26d1c6f88a7a3ce735da812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2a375d1b01d403b42a8c390965ff9e221dd27507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2b839d5691a9b6fb1254449d2149753eeb015fb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2bad469ee2f90a648fdff712ee3277010c3c4b34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2d15f559dc85020b5ce8be9b15fc7590a8ab8390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/358c9bb8275a807201a0908e23833bb4737afb3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/3975c3373cdf0bbd4cb52660309bc322936d8bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/3e9a8efd9c9bb9da333b51639cb233e262c73e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/3f0679545ad58adc51cdc28825a3d35081bf7208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/4996b1525550fac0b2c3c1c2467c05c5c3a428b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/4b59eac56182b54b6ae943e4be3f1a458fe81d3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/4bb19ce01ccfd81beba8382e10d16ffc8a55a58f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/4d35a3b6c67c77d72b6cb6b15727aef2f1d1dadf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/52b536a775eee9d25964c6838f784c09e24252b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/574c143a5e310cea54a7f218fee8dfba1b04a1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/578e65170136d902d61da7b2004f3dac8306cd1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/59da9d1aa9db9f2dd90e6459ef7995fa1ed7d937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/5a0148824222ce92dc83ab7b8d76825bbf67985a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/5acc8f97fac02e2ce29a457a0f810498df262085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/5f6845e8d47325f44cea26ff438222f62722f247.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/6047da21cf9eca3aa5b7ef93fc47bb32da04ab93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/63559705be0ef993f24c6d07a8205c22da9e41df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/64ac8e08592e19f84adea8141897c66115f1fedc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/6975452b748edc7478bf1a832a2f64b0eab5e173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/7045ec3196ce9cb88bc2767df082f38d5a58ece2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/71eed93d0d4e37d469b3a611c2a11b43423c20c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/767a4f981517dfb128b3ba51a8acc09a90f74c72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/78969b4de73f762879bee3843a0a437514ea652c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/78f5b8dc5d35e594898cb40237a54aae6ca5c31b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/7ac552cb4703eb1409585cae597265720594da0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/7b074a8e2e9d1584cc36c5bf99240b0c6cd50c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/855f38050a301f032ed3560f761afadbc622eea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/8992606395345161e1a31d06722122c4380f2cbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/8d936edd8509b82e241ae52e5c312d5326263d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/8d9cd30a584cc547c3cb91d7674e810f4600641b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/9074e2dbdad78ed17872e2f669a714987c854472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/942dc9509756443b307a8804fe5d938f02349a9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/a1fac1598fdd1df4cf51b3a6888ee06b025c225b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/a414efe5be6fe4b54c55be56f7d009ae8ca0d567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/a68c8023116447ba14bef2d5f8b213fdc57c87ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/a7150fe4c3df17f972cd937713529a238c7333b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/ab0d73f704ddf9c31bfd967772b453d6849af231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/ab753698e00f3268b5addd55b01989eea4d62336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/abb644d84fdb8f149723c5729e2e67581feb2e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/ad1a217a432b9bcc588af949049e2bee870b5ad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/af5081cea758e1717e02aa320ce02ed23a995ce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/b055aeb284e0459205068298ac7dbbd5f6423523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/b51502ff0968ead4ed835f3606b9959142893d5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/bb167d052c76041269b835b3bd65c0be85adacc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/bb5113c45defb7190a44c86053fc11e31a3706e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/bce00ad2ff4d12cc07bd40f075a7d577f02da543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/bd7a80ef614ab14d58d1ff39611533c4cd9d5eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/beab53c06450866efe908c98e2a3475f29eea7d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c0345a3d83a8892b2681b7dd5c19ff0bd783f5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c0508f0f5dcf587329e46ea89356999f87e11afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c24317d8365ad034e6a3de9903f8bd1beaf35d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c2c4b3f2293cb989f4768a280a44e384b588b96b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c48700585a8251edf1d201dfa6cadb37606ea53b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c4ca4dc42c561e5a6e4ea10735ac2baeb3d58cd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c60498603b072dc4507248c4e804240cd51a78c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c81a66558c2579617cf3cbd605f9afcce7933f60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/cc4bdfd86ec3c8d4ea755a6d40ed94399410ffae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/d27280aa5d1b3a138be03279203654c0a36eb584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/d3f6e2c0292f048bf8a994bbfc56eceb076ed243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/d6faf7a0cfed05b2d68f68653c713a1d5f4c4d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/d99dabd2bb788580da602d2dc4c69b1c5cf6f01c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/d9e627212c882b1517e67919842df55077493652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/da381e6a0aca8b5632c207b7e545d450eb2b3108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/db5005757fd109e6b0874ca3fdf450d376b00f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/db6d9621f907e8a34eedd1ad2d986b646fb35ae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/e7beb8a6e19dc097ab5ede4238c665fe77077213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/ec1a4b6adf6eb1bae29a5a357f43259bdc81164c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/f3e40c890ed03e3da3e585165050245603466c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/f4fcf2ec2027064eb4561b3fa77818683302a8be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/fa5edd733162ef33ee24efa1297b4c71ca48c744.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2016-22/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2016-22/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2016-22/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2017-04.yaml b/configs/cccc/dedupe-url/CC-MAIN-2017-04.yaml new file mode 100644 index 00000000..e4ac0f75 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2017-04.yaml @@ -0,0 +1,599 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/04da946a14c8f871e25dccacb1c94c2742aa8de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/05e7e171627362d64902e43fa1156b28fe06688f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/09b102d0bc9716726971dca1befb221dec351440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/0b63dea6e1cf95df4fca723be240a84cff40faa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/0dbce553bacaf7c6f18b78df783a6e03318222b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/0de1435023ba0f74d00c0c27aba0c4a0241deef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/0ded2acac2484b3e8fee9c0cc1b84badbc8b803c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/0ef536414b4bf3aec4597fc6ded23219d795d323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/103036c86cf46553dabd4f7be1d7a09686ba7604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/12805b964a5dd868149aa697012a9fabbca15046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279169.4/warc/66139f14c3b3f67fa85e19f4a107b06f5d17e0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279169.4/warc/6e105481e714b0ac47ea5c5ccabd607795cd6956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279169.4/warc/aae499c67871569c5b12885be2b1b741e77bae42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279169.4/warc/d4c0f258203168cd79f1f31625021033664adbc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279169.4/warc/e5d7f9693ef2e6faaa8550def86b91fa9db19c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279176.20/warc/39efa8a97160015d55c226d69ae4d8bb7664aa3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279176.20/warc/68a7b952226b3041a9662da6aa9cd83d71b1b8c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279176.20/warc/9caaf0bbe0a938fe6e5a3df30aa370f5df95a74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279176.20/warc/d31b3e5e75094b25e83148786ae8b0fc191fafcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279176.20/warc/eb92f2094062f453521914718a35c15c4e6b9bf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279189.36/warc/200d09f58cdb88505bdc56e765d4a0e603439212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279189.36/warc/5f0c142b75cb3988d7c427d8e44e6b15ddc17207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279189.36/warc/a52a7305571544054986433d59b2c0f404972ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279189.36/warc/c90145d8c1b5f805f267a9579887656e1444f02e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279189.36/warc/dfd0b811d10367a281f3ab9d26a4cfda91fbda93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279224.13/warc/0e10ada819072d2fb35f6a71cc7968f81f8f70a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279224.13/warc/1f5e168aefa7cbef33c2161db4549fe4a3c49b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279224.13/warc/b0cab49aaf28d15b4b01878e753edecf21ae5a23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279224.13/warc/c2caea12cced72db1b4c6cdbfb6b05f660f74482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279224.13/warc/dd152e6209151a6c7fd743a1c874835467649376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279248.16/warc/2a051edde3bee43036382cc2425fe01b18948b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279248.16/warc/4b322fcd47270f00eeb5b61e4950b92cc673363f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279248.16/warc/7aee2e057c0887dfc5271246f3f705ad68210b6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279248.16/warc/c43bc7afe7e4e76ccf2d3a81dbff4a46ef24e49c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279368.44/warc/360bfd73ad50f48bc279145e8db6e740a722a8c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279368.44/warc/42de2ed1428d6f1b446c0b06d775c822fe61288a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279368.44/warc/7b6f1d0727511b77d5bc72502e6190e3a157bb89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279368.44/warc/ba0e03c57b96e6c1a576227ec2c85ba1b0eb5c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279368.44/warc/d37032a1a8dc2c9804c931c4cf283a960f58b2e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279379.41/warc/2a00e132988af77e1aefb751c9ab5987ce5a87bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279379.41/warc/3912b12dc3020bc4e5030737ca606fa765664c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279379.41/warc/4d0c7badf6700115a557e74ef7c2c0dac06d6e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279379.41/warc/70cbc47262f29ebcedf01dbe8dd0fd9e13b653de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279379.41/warc/e86f7162108208be8f51081f79f53b18a8a413e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279410.32/warc/4fde12ee41a6a1477981757426d3393afd1f0e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279410.32/warc/80c38949c3b7175459db73ef0114c3842a7c826d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279410.32/warc/81e3c1855f594b322d2ef6db1f2550c470a21e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279410.32/warc/87eac9910df43c06570959b23013d3c545edfe8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279410.32/warc/d06506f8ff1edc1b226947acc6d986a2f99e2937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279468.17/warc/3ef19f8cd590cbb0cde67eb449d3cdcfc49b5cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279468.17/warc/4e6dffa5d5bbb9c3572c817d2fa3f0fcf280d5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279468.17/warc/683e72f7f714d7e5f5c5a4c7c1d9039edecf1e0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279468.17/warc/6bfad636274c3f4e077d41e41a404037b7043be0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279468.17/warc/c0b34133efafbcf9f876de0c38c713570b91cbb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279489.14/warc/179cd0246e13961a9213ed68c1bef92349984f2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279489.14/warc/4158eef39ab47d140fdcfcacd8394976ccbf1f51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279489.14/warc/7cc01b09a5c3055a74f194e3b956062ba39507f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279489.14/warc/a9c2b14328d5b61afa4e0299844931888de3f1fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279650.31/warc/3f4d4266cc05dfc2db5f997ab8a8220d3b5c6f8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279650.31/warc/5c94ebc5082773cd90be4c7247f9ef0f7d4f67f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279650.31/warc/64b4617660ded78eec7a36787a3a29a44b26e20e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279650.31/warc/6e6c37903df9d6b86db30d049afa3fb981a0a7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279650.31/warc/f49c263dc21a353358135ec56b58be18379e0811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279657.18/warc/92ac5869f6ab9bac9358bbad7554ca4ffc0e480d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279657.18/warc/943ddd2c61802415777c312c202bc3d762a27241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279657.18/warc/94a421ba90699f1730a29aac8c3783fba06ac1b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279657.18/warc/95ca70c1f2398a208c85f2f327771515787aeb45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279657.18/warc/dfd0d62b0ccf569f77654c79555db88ae2a948f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279915.8/warc/3d5a03d045d37f45d73b3c281a59c856d4537f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279915.8/warc/4907945bec46ab7075ec2da0a246b9fbd19e819e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279915.8/warc/4b223c84d7019893a4e951cb76def9c2e38b76fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279915.8/warc/80e85f55fd82bd6935b38761df715f49c45ce2ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279915.8/warc/fc6440870d62694063bdaa5616c1434cec7c4b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279923.28/warc/79dd26ae288ca91b7187bd9387e3b2db9a8e0bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279923.28/warc/8e02df4237bb570a2f4be62db7bbfa7ff40fdd5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279923.28/warc/a7ed342ecff6c413f48fdc8ae5a466b4d65bf048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279923.28/warc/a8dd2d06836e5b133615422da8dfb222f02c5f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279933.49/warc/28c13985f76ad6114e7394c1bf79b790bcb5788d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279933.49/warc/68cd5c6a61033cde9520e0d10002e6879f0b1735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279933.49/warc/a57480639a276d30557edc1dcc989c73955e2069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279933.49/warc/ad8e20d0e8f0a215ff75f451e1f9f2c87d1dc2dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279933.49/warc/d23a63fcd590cf804dc33a45ee874eb3e593ab18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280065.57/warc/4fbea3db9325bd34dd8dfc86e768c52b8df4a135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280065.57/warc/982aa13ee4c2aac13f6a13365308818cae7b91fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280065.57/warc/9c47bcf3ceeb0d542182bb11ae041d176654e4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280065.57/warc/9d948880d72336a0b05c23c701725e0b1ad4af41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280065.57/warc/eae0cacc82129e0ba6c163e0e9ff4076760a5ed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280086.25/warc/215425c902391d37d1e30d7a67d7f3770597c819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280086.25/warc/50965da5c11544b85b1d942e7b3c6fd9885c9d0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280086.25/warc/cc7f2353d1a3f8133fd849755d55f689e0735594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280086.25/warc/cce38731e78475f1d8dc4a1b57f7c84bf2cf4b95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280086.25/warc/f25f84ce8ba823a52b32ce4f17ad17d366874aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280128.70/warc/a03d17417c828c43cdb1e16096d6678332b24611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280128.70/warc/a376ba76106d590b90ad21bc2019f30087ba0f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280128.70/warc/a9805a72146ea3fb929d557764de9d07c7c6507c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280128.70/warc/c6b083fa19f7dc9af68c6284b3c691eff6945926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280128.70/warc/d1e4fe966f83ea194bf0a27b8822253808ff2e67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280133.2/warc/307342726998880e9086405fa486ced9d76bd435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280133.2/warc/3e6fc2e84340792b0445308da63b10fd06b084f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280133.2/warc/8c24738f2f45aa93563ab423441bc3615b4c7bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280133.2/warc/9302a4cb2255f7c12f412ea257e6b622cff79644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280221.47/warc/15e653219737b1b5879afb1b4e926f54c1b4cb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280221.47/warc/38dddca55e2ec8fbfdb311e949f8de359884ff55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280221.47/warc/bddce95a58762ee93e5b482859edbb5b7689d850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280221.47/warc/c44b1e7320c2a4d0e72ac6e361f4c79c6f755167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280221.47/warc/c685211fa52e745eab5a6d25e6326aa351613b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280239.54/warc/8b20a384d03366badd016c5bb10f6b6691f5eb65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280239.54/warc/b204453bc17f3913f845615cfbd9b2fce9051e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280239.54/warc/c26ef64820d28d5cd4bf967fd263e3927ef210cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280239.54/warc/d3243128179e5297623ba4c1cbbdfe3ea8e2cd16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280239.54/warc/deb9b3f10364c615eaccd9a5c6e594c3d7544f58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280242.65/warc/73191a894d4539393ba95ad5ef5615b69cab8b38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280242.65/warc/bb970a6ac4c7ae2eb2f7ce45635615b771614bc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280242.65/warc/ce79d555f284c1592501491a47de8e0f2d6ab6b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280242.65/warc/cff01a6fd4e0d6fa2addbd7ccfb930de2772ac0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280242.65/warc/ebcb2d007d8c26d1763d9e14a0029af892ff9f33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280266.9/warc/0269f661df266c68c71b64da837a3a59147ed4a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280266.9/warc/06aff40c5113f959cc98e8a26e3129ec9880a2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280266.9/warc/99d6d557ccd4292c807d1a9ed214e74155e87e32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280266.9/warc/c24edbe6845e80297da8ca3e8c077b3bfb3a1b7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280280.6/warc/8493207ca4b0c641cb97c5a0be07988beed6f414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280280.6/warc/89d3f6462ea2a25198a2daf907a946dcec2fbb8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280280.6/warc/c2351d94fde428ccb3a8a3e8801fced29457ded8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280280.6/warc/fb009771340d568ba39725ea8e5e6346f18b3fb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280280.6/warc/fbd9d4cf530a30915f5f7992bd436d0c2bfcaf51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280292.50/warc/0145ce53c59cb8d06da91767cfdc4729148e121e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280292.50/warc/0fc9de1fcfb677252b2cad8d885ac7086f6f7f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280292.50/warc/364d115bc01a74c3a21422aae304baa5ad234c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280292.50/warc/49fe6f531fddb82a40145c15c2b0cfb415c5bd98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280292.50/warc/7511bcded3ade9dd273ac38e2405e2549da5fcbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280308.24/warc/35805d6f670c788d444fd94372f287ffd867cd14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280308.24/warc/5af4ab9ab263c44ae49456a52450ce56db7dd376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280308.24/warc/6e58024d19c664543152fc0bec8ad44aca593216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280308.24/warc/7713c38f887a934abe94a1071a91760efc5830e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280308.24/warc/a87e7be97dd7dec97d21e80f1c4a8d46eba30db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280310.48/warc/7b89b4a54bd1889de8bef4b56d3f096d233f487c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280310.48/warc/81b4e69ff00ff1a986e73d21c1cc36560d3bb424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280310.48/warc/917160276326692a075d395759d012d882ce8c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280310.48/warc/dccabaa7f9cbed041a81a6e50b23ff1e63885251.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280310.48/warc/f2f6f57eea36142d4f7225906699af8cc2f9776c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280319.10/warc/51f5da723bc4db2cfcdf460571696d45175b9421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280319.10/warc/676278a6587628bbb67467558150042d41608210.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280319.10/warc/7d2aee512b0991d67827630604bf45729c0e4374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280319.10/warc/9511df54496e80004d712ab129f4a5b56aecd083.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280364.67/warc/08889d54b7caa5dcbecab0533881688bfc629f0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280364.67/warc/544fcdcbedfc581b98a56ec5ba8a4be309537da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280364.67/warc/5938f29a0132878f0f56fab6b5c3307fc82016d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280364.67/warc/afcb7fdd788a515de6375963d55a49151e8c6e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280364.67/warc/c186fff489bd3bb7e1a31c77f42c00a16507aa54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280410.21/warc/245460d32870519ed80513ba319fc300e910eed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280410.21/warc/2a638cbb50cb3c56599465023ae92df6672ce46b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280410.21/warc/300bc013a46b48363f085ab3eb481c8b257153fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280410.21/warc/b1bc782af44e3ee1e681dd1420c2b8df6e913c10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280410.21/warc/bd2c6787f7848d426bf241fa9a9c45d06119f2da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280425.43/warc/1afd75ba02ce5a855b94d7502ae480cf09028bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280425.43/warc/1c6869a4666ef5933673d7938f166795319b18cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280425.43/warc/281e9014d3f333357a6aa2603957a4aa5c1d939e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280425.43/warc/7e16cf68ca6fcbe9c1f17f9332ba44eac9b76f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280425.43/warc/ab7b788f6d6c0e786c8e638ae92b8c5d86b0cd58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280483.83/warc/9d3938b73376e96f7e5fb2fb4af5c11c218ab882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280483.83/warc/a6df55e2463412eb0a56e2b328040879a3eacb32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280483.83/warc/f1617e4cffb2a11df16a797fdd35ceb151372908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280483.83/warc/f34cbfbe097dbc712d5309603501fd69a5240b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280485.79/warc/042dca2541a934da4f3961a47d940fbb7b57ecae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280485.79/warc/d3e47cf29544c44dcd4bb8b0ad2fedaa5ee32b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280485.79/warc/d4eb0cf88e88650f6fe7dcddeadbb29e89ac80a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280485.79/warc/f5b0b95d91d6202783156fc1287d23b221d11166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280485.79/warc/fd96e55ad77c23eab1f4b4e096c461872ef444fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280504.74/warc/67d31e335f958de998df114125b91d968cad52f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280504.74/warc/87d06347206f1d548b75cda96cca10afa1236324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280504.74/warc/c225720624730af48e4164e47bc1d17b802061df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280504.74/warc/d94b1d0ff9746b50e5dc20a3f9a30cc6c59e3185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280504.74/warc/f33e9272e957b78fcc9432ef163f9cd28d538c1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280587.1/warc/6993bb0b5cfab18137a8846364c75c13351c15c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280587.1/warc/947741e36e87343a8361829d65f6d93224a4b4ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280587.1/warc/adb1b5392fd47b183cc6b7f2e77f89cad8202b7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280587.1/warc/bf51046eb9fa4069ae0ebb2baa1d106d7524b924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280587.1/warc/f46d9f90825e119313cfdd6b562f15e2f83d9d6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280668.34/warc/4b2805a8f592b288d033f6c9c6913f4acf44ccfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280668.34/warc/8a331eea3ea300497ec019e752c3fd589e1adc01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280668.34/warc/94379645d1c00db95d4a310862cc2250e42e2668.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280668.34/warc/ae43bc765802f3761233e50ed46f58a4d5f553de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280668.34/warc/bb75d4c16d575d9a6f5d168bb1dc01c87e1b9e5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280718.7/warc/021ae30016b55d8bedbb1bfa0cf5a36f597c9da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280718.7/warc/050e7a9c2b932673fa392ee26f7f885b0fead201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280718.7/warc/92825d70ea82e62acee731ccdde748ac8052b453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280718.7/warc/de95cecd7fdd705ff0848699d16f93a4e4727dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280723.5/warc/392daf9cefac49988f0750d938888e6911845055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280723.5/warc/667ceedce9aaa44fa3b42edeedad28c6c12c936b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280723.5/warc/6eca01cbf2acf3cf8ff40eb591d7ce66413d72f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280723.5/warc/7ebff0a557cf31e4406b664185a4df1da843b84f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280723.5/warc/a8e3171e8aed4ff5cd904366490ba062046a9cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280730.27/warc/317fd4e83e6dc4fccd38be3d2a3f7bf5f3716098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280730.27/warc/423f92cc582034c0de0b41ea2378a623c00ffbee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280730.27/warc/56c19e748d69d730b1a62e018dabc4aa425068d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280730.27/warc/93508c6d00d4be269d9647c7e8101bfc020e259c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280730.27/warc/e6fa391d4cd32e8ec52a14eff14c85dc06065dbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280746.40/warc/1718f178774a5d1aafb17b0d6d432fd8145b5756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280746.40/warc/762012c465b8e3ee1199e1d7c44e4b9cbed6dade.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280746.40/warc/7e3114135693fd8598007e8dd25e0a63c6d6b122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280746.40/warc/90bce885e7d4b3a7b35298f0741c0cc5ce17fd47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280746.40/warc/a12e6d811deefbe7c3f290d163558541470a18e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280761.39/warc/90980ce7974390d3f9a9c43dbc2d484880026662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280761.39/warc/979d08f15a8155fcf8bb937026641b91366f6560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280761.39/warc/bc9629852441ac1deb6cb608ffd8e095b76f402f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280761.39/warc/d8b05fe2f392603a796893dc1bcfaa09e81d19c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280763.38/warc/303e9c8b599f563cf64fa8d4b4b7c48016223c63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280763.38/warc/3a42f5dde75ab2d798e8226c5535739a5eb039b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280763.38/warc/5eeac84417de81b709d4a8a6bf9ee18bd9db1dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280763.38/warc/a1ccad1fdeaf9db78b204963ac148b1e5a1ca155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280763.38/warc/f2de7d604b1eae174fac26b13a279d0b7edc2256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280774.51/warc/17bfea445bc88bdf9e94d863d2086733122dffce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280774.51/warc/3e313f73f7eda2c58658fac78020765cc2388b0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280774.51/warc/46a139717663a2e9e51def9c2b9a8663b7a91665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280774.51/warc/754e72cf3251c0b3ad8d93444b78d455937500fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280774.51/warc/d54611441996893f3f6cca86bf0bea111f71563c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280791.35/warc/1cade1540ff28ab2cae9b08652cf6bb79ac67ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280791.35/warc/497d4b9bd68c17e59674990b3c1644bf2ba49d54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280791.35/warc/965492074a4244d5894a224c89f20818e508078a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280791.35/warc/b8d633c078d771fcb684eb441d643f9e37f0c2e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280791.35/warc/cd1da314fd53b5c46831fc34ae698df34c356622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280801.0/warc/0cba29918fe9d4b9327ad2f5d7ae6061cc1e7486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280801.0/warc/353794529fbeedaa7849e1f7efe383920854aa4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280801.0/warc/425c24286cb4076d17d240ba2d0b43bc4c3eeaa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280801.0/warc/432ddcb86d4337cd7ec47232dfe30847e37bc826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280801.0/warc/5724702fece17b44bcbaefd3aa90a8ff32533b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280825.87/warc/996e948b944533c8a574cdd33099ed7295cf0523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280825.87/warc/a9d957c85315553cbb1b13e7fa8d5bc7487d6f6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280825.87/warc/b458b4095e241f0b6d28ecdb6746c46d0c098e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280825.87/warc/ea18b92421e46c546f9aeb6b3d132e9dcb6bf8bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280834.29/warc/24442a98804c8005ec6d04e2281a463fdc54adf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280834.29/warc/25feb4c84cc5c77bdaae7af1de31136ef855c0ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280834.29/warc/499a68d42d9343e9f8e0204709adb1c8f193e4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280834.29/warc/897efd904f3d9cbe4be2a697c6287467442df9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280834.29/warc/9e6e9931beab1797193b03c8e2e15445d94fa040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.22/warc/4b46620afcc62137d4bebfd5b7b665dd10a31324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.22/warc/b4babf53f72dbc342837b0fc0d37b2bf72165a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.22/warc/b567b184981aa4a26b6ba9773ce88de5a9cd66b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.22/warc/e6556b72dff5aef402ca896fb5833b096508cce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.22/warc/f531de13def156f76f1c7de45e29ac5017f6e756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.60/warc/17aef46fca78618e160deda8263a054056d1b472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.60/warc/4e52bd6baff82b6ce523c1ae16300908578a74ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.60/warc/4f6a5c3676219aa83768f8504c60740eef1abbf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.60/warc/b7e55e34943def1710a27bc80f72660091b23b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.60/warc/ffde63cbf9bc1f40fec9175224470f24e496064a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280850.30/warc/0aa306e27734c3b51b55152c081125e198b4ef56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280850.30/warc/0f894f9fbcf16bc738cd5f4c9160b7ff21b83702.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280850.30/warc/278153959b9a3665c13b98f0094aab298e63c4b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280850.30/warc/9b0f5e426716247bba7b720a92b8ca0b4a41f6db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280850.30/warc/c40440b08b24a50a815e4ef65c3e9a0f3186f05c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280872.69/warc/03bcdc8e43f39c6a4a43f611747d64016fdb744e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280872.69/warc/0cb64626de918c4c2dacdcbcbab1d9251d0fcd22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280872.69/warc/8beb3c2b111a35018f4284f4b61644cb641cc893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280872.69/warc/8dbe99ed783bb12ad5f9d5243388aba615e4c4f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280872.69/warc/9a76b14fd933ce9dc8d97232785673f56baa5df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280888.62/warc/012d37439bee31813289928715c365365e8dc700.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280888.62/warc/453b70837c1c4a59bc605b9f35fa84a087dcae34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280888.62/warc/53bf73a9e74ccd4cbbf51ee4701649e2eea02ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280888.62/warc/767a7ed15fd0963249188f478b2794972f3c9a85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280888.62/warc/83bf7097f0c971dcb60ffd6da4fcfbd67f08f524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280891.90/warc/5a2edd90332e34d312d35ac73ff88377cfb9551e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280891.90/warc/61cecd11d1f5f27a9e8ebb69f437e256e1b28baf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280891.90/warc/7c8ee7cd892538b8e70e3806ccc0142834c899f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280891.90/warc/80e13c67cf75022fd380552d25093ef056b8561c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280891.90/warc/f05867cc9da5a19a02a706d8363cf9c66b4bdcae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280899.42/warc/50cfb8aeb8cad8b303cbcd642b25621339fd944f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280899.42/warc/64f0251d15100ec282b8210c05a330c0cfed4e1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280899.42/warc/8c26daefec46a171caf13a58227b11533619727f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280899.42/warc/a366115141b7f70dd595921f25ba3f9d2f9c7f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280899.42/warc/eb58164bb454b76cd725a4ce5bd0c92507c0f4e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280900.71/warc/8245fccb1009e693c1349ff8f8b098a6bfd940dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280900.71/warc/a3c162e74ff1117af422e9cb8d2b6f582829f894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280900.71/warc/c97a2e214cf9261d257510ae0fba8d5d9ad2adba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280900.71/warc/d0b21fa6596a7d33fc01d9e61e8e8ddc99363ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280929.91/warc/38e850d20b19c396ee406583560e4e8d9de30eed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280929.91/warc/47a5fc5eaed0154c3cb20f00cb35ce882cd8169d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280929.91/warc/a19b4d9d4eeb4e14d2c3831b8962ba6b83b202b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280929.91/warc/b9c81762b176b42b27f1dc6c3e72f00de980a312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280929.91/warc/f2ce4a1d0fa728199173b884a082b7abc1dba198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281001.53/warc/08d5e435bc3bb26fb2cd6da7d87607fc743c2243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281001.53/warc/24ef64a986683769b3226d0e656ab7546da9ac36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281001.53/warc/398e63b22c5c319624f4640ed5558f12d4021cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281001.53/warc/4f753ee58b35171955dd119069449240398527b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281001.53/warc/56fe6e93da46bd8d2f418a307c9673a099f938ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281069.89/warc/12bee887a2b5657ad8e6b8e49d3f747873136d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281069.89/warc/5524f5081782c338784c438fee026c5a04d988c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281069.89/warc/7d343003619b4465fd5c33e697330ba3e4dd87b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281069.89/warc/9e94fa81c5bc0561f1f5882f862eb5f93eb55c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281069.89/warc/f69b1536e3575c7447c4a3da019627499e99b060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281084.84/warc/501ba4e9f2bf0289ae5b7afacf0c02f145d8bd67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281084.84/warc/7e68844946f577c32051a92f381c7604facdc7f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281084.84/warc/80bd48d0d83bd5255dbc262a71c21e61be48ce2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281084.84/warc/c492d786369ae7a47c8549ff6830ed0ad483e87d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281084.84/warc/e6217e397e482cb86810caf208a2d1d5d5f1d508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281151.11/warc/3f4991aca3180a53c7aed0f9089e037f10f9378c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281151.11/warc/6c42745f026c9536d37af08e77b9492979acf907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281151.11/warc/cce1bf9ec307d6a92dee07135ad2c79a5044f7d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281151.11/warc/ee92c3c9da3892f6fa1bec2f7302b8b50f4a542f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281162.88/warc/1fd0953d5dd9e0c8027366c49612b1af13da7b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281162.88/warc/33663c08546ff6428fc89dfd019f56e05a5c1442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281162.88/warc/7addab39a7181cfd52d147287c318c242105f508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281162.88/warc/d13f0e49b1ab5747bd1f9f12e88702d23794b548.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281162.88/warc/f283bcba69b8531625a9c776d456ce0d4b7cc9a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281202.94/warc/1c68acf55375e1a8e5a16a2007f7f7c561e7a068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281202.94/warc/1f6bb5fbcb5a5486d548d82d45d803f6b4108ed8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281202.94/warc/55c938b811eb76c5b6e242e97003bea570646a71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281202.94/warc/a328276c33942830b8c9ca729d32e3e78333c3af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281202.94/warc/dde038a54a3c32770e21ea6cd759e9d814d2bd72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281226.52/warc/0b7e2bc5bd1ea788b9c4de7d172f3f75189a108a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281226.52/warc/5414bd174013096ea30521d9557554d95bbbc226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281226.52/warc/78a583a5aff9c68c87347efd51f4cb39898798a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281226.52/warc/a865667723bf7209cf86c572200999639d56486d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281226.52/warc/e9372ca45a4f370b773a32095a61ad841e288afd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281263.12/warc/114a153228e7c7865da40e5e129a73727555f5fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281263.12/warc/b42950ecd056577052438d47689693a14c55eb10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281263.12/warc/e4e12157cc70bf357e02bf747b91a14caf6be6aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281263.12/warc/f09afdf5766da4182b3ba97d9eeb4add0b4c2eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281331.15/warc/3e2dbc468d8d72530f653a8506db8a2bab462b46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281331.15/warc/69f1ccd43fdfb9a12d498ec2bedf54f436d8f830.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281331.15/warc/89d56dd4f1b22cc28ade76ebebc90cc6c1910bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281331.15/warc/c0b7a556978aeae9b1fcce3eebf0d7eb25a4edfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281331.15/warc/f551b08fee295b6b8e2b06f4e9617f9a84c7dddb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281332.92/warc/0bea6a97f223ec4e06807d7d280fe46d56d781d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281332.92/warc/4839bb9aabaf9f168b3a0bf65f77ecfce7a2ef44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281332.92/warc/6b0c32283a49eff762856c7d5150003da2e6ac68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281332.92/warc/760bc5809fe98c908f2f3d1221512b8ef79b8c21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281332.92/warc/d7bab5c4ca61480240602273c442c289de4a2f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281353.56/warc/02e39fc73ccdebe263d1530e5151df83060b6935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281353.56/warc/644d26a07df96d5b62c075bb9c73dec27c8c9e12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281353.56/warc/85c573b44161f3322686440749315d9f9510dde0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281353.56/warc/e6c0a28d7b748d67cb32752c4f66cca5d4cd9848.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281353.56/warc/f0519e1a9f9eba6d4281e483dc322151b910a238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281419.3/warc/5e5d50a46e83ef53b5dca09c178f41e816acb1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281419.3/warc/7918ae8249c5023158ca6d683791113bd23a28ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281419.3/warc/b9bfa73d20afe37e321da309dd4323441be2b98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281419.3/warc/d966e456120f6d18a7c0b7d33c2b76fe7af362c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281419.3/warc/fcc2d62510fd023c588dc2903bc390dd29073516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281421.33/warc/328ebeb2c6a82b451fa95e86fb7e120ffba14784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281421.33/warc/39d0519fc6f4f53c3bbc2a0184c036c8ce56b9b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281421.33/warc/b6414db87ceac8a88eb8fecf8330d95e342830fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281421.33/warc/ba13d734d0ff26b2eb2e94c539b39c60a20fc9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281424.85/warc/0a6450e4d1b5628cf73aadffb9dc69a767b5fdc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281424.85/warc/50b8fe744181ee6b9107a23aace636a69c3bab66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281424.85/warc/e179349a18fae62146b35d4aa46493f79ea2a4d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281424.85/warc/ede269823d9f831877e2c8a157a7d4333534a6eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281424.85/warc/f7e1c73df8570aab055c5f066004ed2199df30e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281426.63/warc/7d67a0b557d86585dd3c23d14df322bb9b82e603.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281426.63/warc/99120a0aa1f562b10b70bea6efac490e1c7e7028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281426.63/warc/ea6b454d498dc982e952a8f5f28df3f76836bed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281426.63/warc/f7f3028c4c20920e4eb4736187d98c430c15df29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281426.63/warc/ff829c37b4491d9e4bb2bb07b7c81563c19af4d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281450.93/warc/1bb5329eed56cfc81900422ee52506f1b29b2418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281450.93/warc/1e3f41a2a30364dcd44a0a8006e9c3d6005400e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281450.93/warc/300c5cc52300c485384d39313c38d3c59a88f388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281450.93/warc/361a65dc4bdfa0c565535ae00c754be311d7598e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281450.93/warc/72b9319940f70296e1c6fc9693a0972c79e11878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281492.97/warc/465988933920a739f533ddabf2b7ad2a2077eff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281492.97/warc/5aaacac97972a21f6a3d406073ed7354c027f8f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281492.97/warc/9fcaf78b18470af043a21335dd677258589bb299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281492.97/warc/c46a335d539434904bed06894a99c942440d8653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281574.78/warc/212bbfce22099ce208abce0c2965cb2e0d99b543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281574.78/warc/559df0e0d18c087f1759a9388010e15b79aa21e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281574.78/warc/8ea9e6395f031cf9581559cf5b5188323d7b7cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281574.78/warc/ccdeefaa7a2c901de7e2772f99cee70186841f83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281574.78/warc/f9bbf46ee0dfa07fe7aeb3c2b370a50673ecf089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281649.59/warc/0f578403891db7f3966ba85a854d2694bf6e6869.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281649.59/warc/4d9f6ba5d47659d83f440734a0fa5d2b8c84d5c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281649.59/warc/56631e604350736b57c0eaeb5c37e8bad0ba167f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281649.59/warc/7c87dccf15973d82f7e1a0187171145f0cb73b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281649.59/warc/be4d84b6a920e56f110f22a669d17c1d387af69a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281659.81/warc/7be483a7140c5cf116090a3638c55198ffc466da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281659.81/warc/9b03edcaa18a93a9f626f9f2bcb372ce7b793ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281659.81/warc/b03fab7e605d7934d734554a244931dc01ea4ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281659.81/warc/b36c12d447e96d727c88c32760c7e6b2b771575d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281659.81/warc/f2540ba62d628d98fb9f03bc45fbf213a75ec898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281746.82/warc/615261155a36fe89b5729ca9cf860d7311ee5980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281746.82/warc/90235f1ddb3c52abb6a02782a28fcc5b6fa4a1e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281746.82/warc/a121a81d40811e5122490cc5854ba84d089b7637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281746.82/warc/d26564554c4b503355ec00e29d13b327e9ad8703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281746.82/warc/eb835b136b3d133fd9afa2cb41f5b8b869e55030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282110.46/warc/32d8c4611f02c70acb0f5fea169b94bb2fa86df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282110.46/warc/4994ddf746124c25fc42512f2f09bb3076213149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282110.46/warc/591c69b8c15d9214f5fb91d3e8824d6e98284245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282110.46/warc/66cf5f4f8a99bbfb50afe722c0651fc53817c71a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282140.72/warc/19e3202be9f87dcf9ab6509e0a2d26d57e859ce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282140.72/warc/278e6171b8256c852108366b91fd4604733915ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282140.72/warc/5f65d3b7d2de8518c5629a1537d253c5c5db6241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282140.72/warc/81fd24864861c1a7e9c73627722592cc856ffbb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282140.72/warc/f8b8c2ba6bc653904d60a1bb705ba92f969e6ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282202.61/warc/1a28f58953ec6e077cfaff4bf3bee24fa66996af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282202.61/warc/29cc4413c8f58e556db6b34c2ea1566e6985c384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282202.61/warc/92d16263c0a06d4a5343e347430d108a152e3c3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282202.61/warc/dc147e5a1b1cafee80fa458edcc2a3ca8d483c8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282202.61/warc/f672f4e93b42b98a6a3df155e4090783ec4bad82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282631.80/warc/0a6242b07455b80b335fd5b165adb1893eef6b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282631.80/warc/0e66887a012377bd610417a8862957437e0656e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282631.80/warc/14fadea9f8abb0335ac0e396589f210ec06e1c54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282631.80/warc/294284ae507b375467161ee37815c36a7588b9c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282631.80/warc/7ceef4c164ca50c0becf7b6210f0432f4970c24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282926.64/warc/5e390ccc3c472f0ee433358d5f5d80e155942f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282926.64/warc/7242c2ae030e64c05d1b769f34cc88c494af8ced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282926.64/warc/9d5bd4ca47c205ffcc6b13121d048b271d022b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282926.64/warc/dc5d014badcd833d3a28ebaedec098c39c1ca90e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282932.75/warc/7d3e1f6ac46149b3c1a35c49865829876e3ce359.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282932.75/warc/7f82260ff369bb1189e7b7b6d2159f91c4e51445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282932.75/warc/9a028af944f477ad904857f87ec002384b976a4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282932.75/warc/c5abd4357c9e1a63ebeb3f4e8a69535d78ac0192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282932.75/warc/f0836553ecc762a99830b9c497a1abcb3c82e823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282935.68/warc/0fe66ce3851f214f7b9dbd161ad8fbdf019f6a89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282935.68/warc/646b1106eaf6a05e2d213ccecfdc323a1c658d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282935.68/warc/8cac13420c3c30f714f17c976d10e903dcb30749.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282935.68/warc/c28c24fd525d914662da6b5f5b9d68ca5699acd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282935.68/warc/fed116054b23f9dcf351a1c35e227c41e1f70b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282937.55/warc/0a4c92172e81b4653788ec1fc27b1aef6905e0f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282937.55/warc/37a6968faaa268108b73854a4454064b249c5ad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282937.55/warc/7f13382293c497ae5772e595d34a26c24bbfa4cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282937.55/warc/bca8e79c0facb78bdd7422a0af9913c7e1f88ef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282937.55/warc/c168881227c7d41cceaa37017ceb511a7d1f6945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283008.19/warc/0d3b162cbee3941fd79d413a6f61790701d0f95f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283008.19/warc/178437a8d7d54a8f663ae71a088e7bb19edc7101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283008.19/warc/1c9780fa96be68971585cda3fd727e839c8dcda6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283008.19/warc/338903950ec4e9e3914509602c6a4ac35a398644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283008.19/warc/399135ef2b76e5eb2ba245327104563c9f9c8a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283301.73/warc/0f40463d0d8914a2d55584f10927254844408315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283301.73/warc/7cf37ec5b7171b6abb6422752adc7740c623b68e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283301.73/warc/806cea2efe35ca1da70aabdada7cd679f476b28e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283301.73/warc/960d1e17433aa4d2ace74f616a64291b55388be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283475.86/warc/1f8b0f590b47cc98fedfe0f699fdd659bd74b825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283475.86/warc/763daeb241f2beeb094fbff33964c62cb310a484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283475.86/warc/770553b7bbe1894627c51e2f0670917ea92da6b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283475.86/warc/db5bfedf1fe2e0894d1f643e7549afad1d902e39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283475.86/warc/e437c77390be72664240468cd5dae9f6144aed7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283689.98/warc/5171dbb5fa26f9ad3608238fb25bf32c8636de7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283689.98/warc/abc49761ebff5c79012b3724805ea4e07afd2450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283689.98/warc/ba7914a75ab8c96798b9415afa806cb4b8493958.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283689.98/warc/c3c683d3afd1dbe8259984a48170c2c874ff879b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283689.98/warc/e4934dc5aacda3e9b64d957b40fcfa1740830eb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284270.95/warc/84d6389d777d5e62e21f0957b4a7e7535ea8b748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284270.95/warc/ab77d08ded0f088521ac276a676cf8c4033470ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284270.95/warc/b2fb2455485c1e85ecd8038f306104fd657926af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284270.95/warc/bb31d818bd2d2a00ad1069d965a8bba951b523b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284270.95/warc/d369681b6caec41efc97e3eb94c5b91435deae44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284352.26/warc/0e3f5ecf7c37ece157dd29491b097b939348639e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284352.26/warc/33e640cb2e99405f695ce34ac070696d9dbbab08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284352.26/warc/529d64e4c989ec6073498cdc5dfb89865cd9c042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284352.26/warc/d829509f3ceb81cc4554afb0649f7dbb868b8fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284376.37/warc/0a925510d99d5a8bb87ebb856951405f4f596c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284376.37/warc/27693bfd362503e9b0c6c238715dd7b131af71f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284376.37/warc/852ed626709285bd4a8377c52ff94dfb0b25f60d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284376.37/warc/a079350aaa9dab75160b88797f11ee09d77cdd58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284376.37/warc/ebba0317c68960861a729f874aa959a39119ee8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284405.58/warc/32676466218c6ccc477afae0c8271945c3b99c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284405.58/warc/8b9f1238dcd2507a5489e08cc172c6ac76460c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284405.58/warc/b97513654d0d62174456ef29f094bdd8d9f21b06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284405.58/warc/c7126f31ce2ca9a0b47a189cc80dae4f7a509c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284405.58/warc/d3897506e6da5da86b4456542fe4021135e26407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284411.66/warc/7c2aa9f02684463b80dd5aa944e34a0c8bf96a09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284411.66/warc/8b58a29158df9eda8c1462511a11a7da6977c9c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284411.66/warc/e5212c19d7cd27bbc247e7f44c7fc4559c393d56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284411.66/warc/ebeb8cef00164c92195fee2eaf02ba44e4ec9533.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284411.66/warc/f2aef9cad40408a23e9f0f242525e1ca44f9fca7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284429.99/warc/1961b0d0f8269addee9f0ebe1d99a7838fe1b37b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284429.99/warc/6ad7fae732d11ad9a3c83f3397a223a35237771f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284429.99/warc/91ab78ba6c65c104fc313dc2029dc6239bfef693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284429.99/warc/f17dc9a0ed40b684d798351d98d1582c3ecec789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284429.99/warc/f63e194c732690ffa8e89f6269ddbde22bbb4f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285001.96/warc/1157c6970cbeac69515e73f60482b1fb55c2583a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285001.96/warc/14b6d0ea7f2940110a70ac55e082fc839325916a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285001.96/warc/bb6ff9c45212bd1256596349ebd50571cc6fe997.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285001.96/warc/f2bdcd0594b7936390f44b3868ef109e85a37cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285244.23/warc/3709e2ed022966d2d47e635d0a4c76f2b084a8b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285244.23/warc/9d8afbec0c262744bcc39ed4f0711663c3285b8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285244.23/warc/a0a7d59a1198ea5ee32df4c0eb9b60751a830442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285244.23/warc/d319c34cb71206bc9b40e293d248926b154ca628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285244.23/warc/d764d70bf995ac15089796f18e542c6b143cbe06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285289.45/warc/30ea5b78ce5fb5fd3e9a0de5807e935e37580895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285289.45/warc/3a60ab75f9637b1c6d55e1405783fe893ba9a76a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285289.45/warc/472c6d081752445a56fabe9651bafa9a4d363a0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285289.45/warc/8d9771914f995daa7efff1405c0e5edcc94f7b04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285289.45/warc/eff2a0e42c91bc9b4185f9b0601c4a58a85eb611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285315.77/warc/149be17464508526e5a95660393b4b81a8db396e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285315.77/warc/26254cea0ee426babaf63ee7f6e107421f8e5d44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285315.77/warc/292fa5319fcb9903e9e4355f2ad987b5578a6d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285315.77/warc/b92b3cb5111e283d6f0f4bc9a8674413a5c22485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285315.77/warc/bbcf5da692d77e24ea4142e841f8a96d77209468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285337.76/warc/2e7dd4a5980232cd3d9c21e0a57510c65406b509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285337.76/warc/368cff2be1b5a769499c6a3375684986d3533e19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285337.76/warc/48b84471ab9307b2e3f4838e9879a5bf9d564539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285337.76/warc/720a991e0164e01332e45241832201b382e28019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285337.76/warc/79f24e346dec1f4eead1c7061ab256aa49e90ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/25288cf33a406e90e3d16c37a55bee22f76a5f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/25639609241e97fe15270cbda9bc6164f0db1b40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/25d76fb071bed1829dd493d3e1e01abfd052a7bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/2ba8f05aee1debdef1bd99315e8fdc9e19251708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/2bae63c2e39de9f254850acca27df4b5d594bc0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/2e583713d6dfc082095f6007b966400cb0bb6afa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/2ebe317a696313e74e05c10c6e5d1f4869794322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/2f15cd957768782593397c969ef80e344768bb08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/33d68d9c9265cf66e0a9d4574a9d539d6d4b27a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/349588c2ca38ec129688d431cd003c2043c21a1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/379ef1b2cec9ca3ba3a87313958957805db74119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/392137954e691a6680614d11a56c114dc3c7d23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/3cf0b971cfe8d75366443c2b7b9ebfdc833a582c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/3ef5a7cc5dc55f6fce7c91911acc0187af35af96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/4542cbc063e22a63ea7002a13454e809ed5925fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/454c2270904809f3035234026aef2c23ad5417d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/4ca01937a41c9ea62c18b571d85605b4d1c09401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/4dab194dbfec71f84b754ed7d4aab9894b457277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/4df8be6ed39c283234dec15501b7724af805dda0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/4efc760e705fafe7c36eeb7d8c0e79e779560014.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/5060b430e549bdeb630d7a2befae06ea117091f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/5902e66a1de7a24bcac20032f151d57722d5956f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/5a9d73340cd8badd983623508381b30bd05dd26f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/5ff607cb17e0fec95b858a4292a31ee778a48909.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/6173b0e5302d3cc43638022d912b0e6add83ed3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/61a45dd230851c9a674e9f55ebc62b79fcab6f31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/63ecd6e5769c3cb4c5a1a4b1edc10ca56b7f55e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/69eea5d0bfe735061d5512ddbe64b78e5b884455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/6b79927d26b7322a42623405be0845e3643456c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/6c4cb7d60d7af0118c844cfbbfd4054a2c800e29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/70665cb7e4ab715e59f134b643ec4ae2be41b93b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/72bec3bf370c384bb936b274e7cf4c0aac46f573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/7785cef3a64f4fd162d38ed9940eb35890e3e05b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/7945894fb4067de4a43b092bb0e72e92689d86c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/7b084291ca0b545059a7f070788b00b6b1b94073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/8132604652c430496cb586206f222191f718c576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/835479a04a2777eb59c4e2c7b3b741cc27e182d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/8507dc1b3cd28dd9f192d241b9b7b7599b9bc1f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/8b72874de2083aeb5c85f86afc1dc1ac55f38bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/8ed9505a949b2531f9ab4de18f8dfc217bced80e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/9289cbcf54026b97d5de19cbcb787c87ce21fabb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/93281af94cbc9eaab5c31d228fa8a716bad39e03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/99d3ea2607ad1515b09e515ae86505a0c5705419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/9a421fa2b3f3298302da846e1c6820f5f0edf2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/9d7c5d8bae6bc6078d431bdfaf25c51db5748c8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/a0089a64aec3bd5086efec992d29e4eb2c3d516e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/a13fa8f3b424d81d5b2fa8e88e97ca77ade86b28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/a9d318b50117a3c12b2e613e375dd70b133b71c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/acdca6a2d1f42a2df275b9b382ae1abe4b5a8403.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b0556404b9cce0316d5bdf461cdec4a22c22b535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b2276c71840e1a3ce87324887d22ebbc761882e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b2d2da533120edd48c8e7a93dd176a291e4fa718.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b327c1fc1af46dec689eb9e82973bd2b8d2cbbf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b35c6e9e0a706128e787c6862fe02b0ea3f40820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b8bae29c2d37485f654c2a052df257241b6a0961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b93afcf84fb32aecd9b26bf739e0d0852f36fc4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/bbd2ad4c176a70e0bedd95c471fc183ebc1e029d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/bc13fa1e79a57e7862c8ce44f955dfc6a114c92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/be7f9419381a03ae757a536365ce56e42bf48287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/c0502f5432b984c5aeedcfae9992890cd01e84ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/c98aa9bb8aa0e1524735401ea9b0fc443341c30a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/cbf797cd250f51127608359abb8b352aa2d79746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/cd49c6e741e933c12f7e218a1a4a15f9eb0ef61f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d40ec4d9f3d752843efb6dcdc8090af8b3ea72bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d46cb76fd0f4b0444a737b4d37c8dbefc18947d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d5f1e43e7eb126894cae3351d4b8ac492ffc64d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d66050cf0d39d2c0a46cb8b36ba91a400f953077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d6a88f5b7562e6e97620b821a5f5d938533005ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d6c18c5b1c891f8426659921bfddcfa429104644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/dc70d58ec9059bfa5ecc8a59b2fce5c836921698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/dde399f0ca872063bc687ff02638b4d7e457dbcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/df592e189743c9f772dd6469a6390972c243ff3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/dff1bece45999c3b1b12b9886e4ac306e7dec2c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/e2cad663571c19cc1ace454e8df29dc9c2a8e8ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/e73baba5461219260a60714c7d1acdd5f2c16efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/e92a60566738ed51d0f2602b769e0c0fe38cd6fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/ead68b0eefbf0f4a2390311a760df3fe6cc848f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/ed05e703aff44f76f264bbe10db274b6f2df0a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f1dcfe74d5ad3cd8a9d3f35f2e38aa3318c8fc83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f3a6043ccabc7dce70162cd5b6ccfe9952a0b9f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f56b8711ec4eefb2fc4f985761fa1ae7d5221e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f584e897eff8a7547e6045761979aed13dc86581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f79b3c3de279668bf9bb848d18d432fdbb2145a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f84e8df3b18f46cb25593f0792c53543809e0194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f901112add25deba56d6805c4ddb20166076a14b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f9b9739e310ed873a4ddb9c1d15480f38130a98b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f9fecb9e01c3d62356978cab4e51c09a46f89581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/fd65b778744d46e3e096631344e3a35aa3afb914.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2017-04/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2017-04/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2017-04/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2017-09.yaml b/configs/cccc/dedupe-url/CC-MAIN-2017-09.yaml new file mode 100644 index 00000000..f228d470 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2017-09.yaml @@ -0,0 +1,673 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/03971464e046a29fa9be0dd75c341a70c1bacf9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/03b9a931e85f397aa54f1ed624743ed9a28d9b4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/07477f8adee09eb7bca6cbec3671ff5e93822f9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/07c416da7f96a29860914012c79b7decc59e3ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/0c2d81e30bda5dfe40e881e6d378011577e33042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/0da68c09a216b4f168b343e79e27873629346fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/0a6de8e0ac19e7cb6b0d20d42c82ad9674e5e1ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/291e12ac82e461731a2b3a3b4e13d6b4cf7f8063.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/80a961b11c0cbd12cbb60edfab01393c51af962a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/c406cfb539dd7d777f59e0a8e4311808e4ae06d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/da160e9d9da89cd46c6b8a266a0401204a190dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/f9d8b53c710d532711a076c9772d37283edd57d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/1d2cdd1f8508d9907507a0974dc1a0a69200b037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/1e27446b31562976878435cf576faff6e11fefd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/26de39ed4587204628d75b956d71c3e0cca363fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/278b80e05c14c0387ee07040009b15424e71e1d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/3dea8da1bda01a13d56458feea4fbb69aedcd205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/a29b0731f62fdad0099f7c232a0865e34b9aaf5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170186.50/warc/04176ae8574ea469ed60fb2d510b7d48e4c99369.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170186.50/warc/2c730c87014b3fe2a689f7eac91db0b2120c3f33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170186.50/warc/547cbacb1853aae7871d6ae79865e6fb0a6076d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170186.50/warc/5b9a15ab9f4ed2c290de9098f46530d556bcee10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170186.50/warc/5e3bc4dfed65b566bf5db7e13b5135458db9df3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/1e5735ec6ce9d4518b385a0f4ac7a59274e8607d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/88c7036950170c3bb30b03e82648e5222dc18327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/b2a7c31c31cd7dbf59a386d086c8d5f70843c0f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/bace42e6d36700fd3bcdb0ea3b2c5d1331ac811d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/d3c2933dbd4dbe4b60476daf81ade67b6739a261.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/fb813bb1087f819c0bc21c8677888719807fb33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170253.67/warc/70f84cbd72b56e22eaaf2b59a38c67572dc02e1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170253.67/warc/9b95277177e4b1f8ad6b0b9a8d3128733e14c8e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170253.67/warc/bcfdc06f595b7bad4bf4189a4094e477370c9afa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170253.67/warc/f9ab1a0a190732d7c1c82467e013b47247e96204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170253.67/warc/f9e0740d611025e6e587db3d20f361da9f68b9e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/1a8b1d74740f6525d7ca24ac83be58da261dc414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/66cb5a33d2c8f447a7aee3586902b20d17e59fe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/6d2cb2b1aa190d9f4db7ff73fa13458d6eb2cfb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/97b10a91ea62859eb6e84532956c30e4e1526eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/9c4d7697a7d025fe2a2741e6e348729a6cfe79e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/9e06deeefd9a5eacede2da54fcd9e744e08e1a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170380.12/warc/49b32a2f3e48e8d7a3f25204c09a9a8e910222e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170380.12/warc/ae48e1aa0794ca4ca1a357cbef5e3317a09b156c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170380.12/warc/c6dd2073b5fd13f40317279d6e76356197a64350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170380.12/warc/cc8958fa58ea56e525df6f4a49c2ee24c692f803.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170380.12/warc/faf89ecd780aa00b23289c047363142080bbbbeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/255c6e4dbd102b30ae92826391368c329c1bdc5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/5d1e3386299a7069fb85352ff6082ef90604f862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/adac68682cb1f5767d433ae37a36990d3e10e40a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/af29b1171a7e7f2062aa7fb160b9a47b275b7705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/e1ae05b6d553028e42ad3f0ba269711297880be7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/ec46da2e9c8945e890915fd40373cdd5e803e8c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170425.26/warc/0c92407fe25be9176689dc2cf43b7368e559b43d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170425.26/warc/232e48b96f141dd94866d61034faa6d88ed2d58e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170425.26/warc/a3e74c32abd45ffc782f092e17aea035ab8b765c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170425.26/warc/cc431a36520fd45b03a578e8a5ac93f6fe7b6e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170425.26/warc/ee1fcf65bf4bc470a8e9a35d6d0a02e87445ccd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/2fddb468c373b4749c0b118118e0e64c30531ae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/654dbd18438ecb4cd50f40ff49b72c4924a119ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/a4f1a55643d9aa94e05ef2b5a20b7d77cffac182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/c573d782445f4b35650cf238080d55a00b95dc92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/c98b6bed78196a039d4852e9ef3be6f627173bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/f1aa4aed755f63ed3aa6a0e8168ca63a5f186589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170521.30/warc/802a5d23fc0b8576d708ebe946c2c9813938bbae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170521.30/warc/896135646e5fd46ac3cec34d94ecfbc50b94feb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170521.30/warc/97348a5687ea01f49e98a1def5640a02891b4477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170521.30/warc/a234b30d83e5bcb01af204107068c199c4442654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170521.30/warc/a577754a4d1c570a5c3a44ec1ae48976650bd2c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/1eba0e33fb9d125bfc16c20c1d500035885e5413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/250a00532ad61427df45384225764c00ec4e76eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/6e93b4f0151fad4937182e843dffcd23ffeeaa61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/8197487465d870ef421563528ee26b467e8ee70f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/bc2aed1f79be806104891d0d7683e91dee8dec09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/ceaf8bb0b86dc57a7430f42499003781ea9b2afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170569.99/warc/080dddcbc8f25675c55b9d7b68dc888b6fa22335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170569.99/warc/6eff5d7d3ea5a9b6f5748d243bd1e7bb094afa76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170569.99/warc/bedf728bb85efdc50554603419a997c3ff43ceb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170569.99/warc/e57e4ea982066bf415296b1a46353a4e2ee1aa55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170569.99/warc/e9fad6aa8eb860a4be1f5879eaa8cc2fb6336b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/237520be52ac2a013c3bf7703cd285c21d8147a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/343724917584d5cfd8832e80156141a262cd0c84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/3b95a347afad55244b8ff699064523e62af71db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/6eec4ed052280dcfa641b5ed5abb8d586be4f3a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/a5a47ca540c4d1fc1050b2f0948fb737993a44ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/d40f04435178b5621d518094925c32d2a84650fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170609.0/warc/7a053555aa7a5c6f9b64e75935f8b85db89cd217.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170609.0/warc/85be3210bd49cc7216ffcf0582a616eab0976906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170609.0/warc/8cc85ea8bb17dba26e244512bb49697360cdc13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170609.0/warc/8f9193d84f17b63ce48a8b86f6c11936dae61d0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170609.0/warc/9dcd65d28a072a808454d908df1e39ff63c82dd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/5f6a2ce42b91d424dfe35e149ec00ce68f3b1d46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/809eb98546d2b3fc3ab02ddbd9c4e2609925df72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/8a8e9840797ffada344025dcc1fe73435f0dada3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/bd90d56fdedf35a2c7ccd60ec6f0410518d37e06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/c8b9fb4c894c0758d21b66fbd5f13d8743f45f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/e72b59a5a72cd3760b5e7113339cdb1b8ba40ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170614.88/warc/0ff47b9a372858d478b80805f241e33fdc8b69f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170614.88/warc/5d619501b93d6a41601e3388c0c062f58549b009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170614.88/warc/77d0e868eaee35482f0a2478f4627519f11d53ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170614.88/warc/7d5450556d864ac3f3d7b720dc71cac4110f43a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170614.88/warc/e6f74e296b100f795e23de77f53de859351761cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/568cffb0de84baee03952c0727ae415e69ca1102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/bfd8be5a3a61cf8758dc63c2292bb331f8a390b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/cb4affb58ea14627a74f9c849b7b95075aa06425.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/ddc9ebfad69127d92746668e7af7ca0de80272d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/de3868dbbc467aa6049904f14db87455bebec830.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/e5884cda745872196d800b80760ef0a4d07bac24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170651.78/warc/0da6d0cc1d34cc823de8ffcf7c31ab475e5d64a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170651.78/warc/42836c3a6b7f9b2573e756f5876547a638108074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170651.78/warc/90472a96b477c6e9dead512d3f1f1c1e2c185527.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170651.78/warc/95468f352f0766c10bde7c742a78adfe05bb37a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170651.78/warc/a6465f132335024d68bb87f3d921773d767e6ba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/4a36d9e6063252969ce09120ea74381ed622b99d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/7a6709257d7325b91fab4d561215f17fc883d14f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/8c712120b3fd6aee7701d508d28ade37627ebfe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/d2b07295fc64768253ee1978553b3ebdf6042e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/dbd2ac130ac97b1c730f87875bbfdcf0752239a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/fb4dc7cbfc74672a955dc256c8f0cf4d8ad87660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170700.71/warc/4da71738420140cc70dabf0f96ef91d043388779.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170700.71/warc/573754af4b68feba3b4c712ad101208d0a1ef2c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170700.71/warc/723272ebc6b39cc6df2529683cfb6d62a6827383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170700.71/warc/cb59f3bf488b85796f9d4b82444a213457fc2d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170700.71/warc/e3d1ed78b87694c7ec1e3d887406b2f065775477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/23dd2a2cbfdf2787db4b72ee94daece030ac52e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/5c3fe26cfa316dde39e1e8573764164282971a3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/a36ccb5f4d798c8ba8738275857242a2b7deb959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/d40dbcf431588485996997e7aceb95b5d5474a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/f7b90c87b434c3591816be176cec0944fe7ecf78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/fb110036af4e12dd8755c8e2087498eb1e4c130c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170741.49/warc/06130c8bc9c338c43845844cfb6bfb59a41b6a32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170741.49/warc/1522774f250541083b64b7eb6a63416ca3ba06ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170741.49/warc/7fb634962a36479a8105a56bc732651047f68cd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170741.49/warc/bb45b46ec9ea1b4d9f5a5925d13a027b4abcf092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170741.49/warc/cc57ea3849481e8a4653120f21fdb326fe58ee77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/0449bb9872840d0cbb92c57fce914e89a36a7c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/0b755680142d43e448d1992c2e351cd4d08415df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/22a5b4c2d07f802e527e9fbb61fa9f509ff21722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/9bf45aff3dc5b4e4672d1f1ad39b5def964256e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/df03dd97815c9e3d10be8e7bbbec8803c41eee82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/dfb5e05b4d920ccade8e4183eee060daa7907dde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/342c9956630b55111f0d27a7b1e471c96a6bf218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/38c105c968fd43e16e6fb524a03871ec7d35e94b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/3bc45b66cbd468f517e35bb31df2be42933acfb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/98d4071576e9464dd09e26bf9645125d05a3e6c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/b807225da2059656b17f8ccdb99c4e4f6565e55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/f8f75781b58c274b3af26a3175393454614d646b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/43e50778f11cb20115d28ea040e9227b402c3b71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/6a243828e01ee955a3b737676401dc411b1333f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/87fc8edf9268f27983aae01b23b3cb0d4ea0dd44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/b7fab904b21a59684000ba886892203f60545160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/b8bfe33744aff9b32be56333510effff26aced6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/deb10fec8ef15a1b9dccd58acee4692a15eddab9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/97f0603dc08a590110f9df9ee9d32c00b47d23a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/98cbd347314dfdbec726b372f8112568e8373da4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/a14b6fec703c56c694c4d03372c97c2edb2043c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/ad7e4c66a7bbeb2aa09bbee41532410c14916ee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/be5a1b1aa6397e6fd1d43417522ae7c370852399.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/d90d8e82f2fa9b20209a5c2133572effc535392c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170875.20/warc/75b9b3fe7d5ec292439d1e0520c669e04207bb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170875.20/warc/7674f0a10ce3d1b07cd5cc8120cd2604a05f8205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170875.20/warc/7bc193f672e55bc40a51ab24064cf20094e96efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170875.20/warc/acecf7218759c52080b49806bc73c49de40add4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170875.20/warc/ecfec3bff07b745e861f4df9426ac5934118398b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/0247aa34377139289bc2e4b7814f02a2f0e42f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/29badbbe4cfd41fa9204553ef6273969d6436bd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/33a3b1c61d0e05a2ae646689ebec3e733456b599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/893548164b4b6d05a9712e9eb0de347b6c1cb604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/af271c6b2a04ee943c201b29428868530eff5be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/f7e06de39753ea7a34d70f80acb5b3862be91f64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170914.10/warc/148f4b9f26a81930f37eac4cecb847f550f8a462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170914.10/warc/3e1da4c5396d6f66dc1d4111d25b3d232cddd8a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170914.10/warc/9a40ad0a68c9890c1b33227f858c60798fc15735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170914.10/warc/ae5d911ebe14338add2b15ff53db93cb3e507731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170914.10/warc/b61b723df617a35c8a0fccbf2693126755c64c61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/2917414d1aebbded10d2911475cd4a9095057711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/3f79f65059d632aaf96cb12553f66594b50124d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/57d74a7af8a0a8211c51f29c11125752bf710385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/5975d5ca768edb93cc04fb9e9b4d6c96487854a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/5ea3d32e00bffaf093238d636778d5d595a1fd1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/916c9195a3de6329036e2a48ad264752665d4fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170940.45/warc/0ee091f72e5f89224857be52c63078f34f0e7c95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170940.45/warc/2b67c67d21e66b0e3c42671de8db9513accedf65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170940.45/warc/9ebf07d6e346b69559e8f2ab0b040eb03bafd334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170940.45/warc/c8f9069120bb0e69601d7ce16481ab2453d48126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170940.45/warc/ce091f289732113329fe35965683d69e4ad9bb84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/224ae118be29048572dcb1774cf6f5d99829eb2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/22fef17ea5ee63aee5955e1c998ec037157d27a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/808f105add8ecda8ec8493532937229cfcf6d47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/92b94f497b7892d189686ba05a0690be17aa8808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/c165d184fc4b550149fecb8ef8cc3ea53071b73f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/e339ce55c4ecc2feff0813ec7d2119d8fb164438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170993.54/warc/0497e180516437510611af914d7932cd5694ec3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170993.54/warc/124d8f0cb41e9fe6af37b11c8121f904a84eda48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170993.54/warc/73711c26a48b6c0123b4283d5cd784e8dc34cd18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170993.54/warc/74566ed574426a199f3a222d8bd44947b024c2ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170993.54/warc/b021e39d6e4d105f31469c7fd136122025b73cb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/02839dae9d81042974321f7ede2dbbbbcd28fa0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/205c32bdc94e3047fdf985bfa9084473fb8f9d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/773272e7e104228d4d5070014f45162fcaebdc27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/9cdefb6548f51d290f8c51f895ddb3641a5eae42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/ccfaba416eba2de797ef78b906702cef794d5e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/cd02975087bd9df3a7bb6f52204750da756660d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171043.28/warc/330660f30b3ed1ca5acb642b771ae6515aa95495.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171043.28/warc/60f8d3f7daa023369a2a7b222e3430ed7b3331c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171043.28/warc/635b81af7c34dca67fb91c8856c15a808c570896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171043.28/warc/71fb0593f50145e2ce84408461d086be9c853df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171043.28/warc/9972a8bf632d042f716013931f3e5239f1718671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/38d228c70a36dd46a709001f445115178805d219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/b0a2ff143a1f9fcbfcb488bf813417d70b431631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/b41ea0be7de1c02df133a75746a5453bac4bdbb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/d9c56a55d51955e421c5c02e74c2127da38f28a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/df73d4ecf3dada3243acea5da2f77716d711c18e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/f40c1f0a3b5ceafcd88757258fe418284d480bf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171066.47/warc/668f081dd5c7d92b1c921292da9f6e5cb1bc4991.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171066.47/warc/b9d47483e8c20c8fe787c0ed29d73c93c7ef6e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171066.47/warc/cdac752fb7f3edd512a33be7bab09284865212d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171066.47/warc/e30d2af2fcc1458ba95d1647c20daed7b41a2291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171066.47/warc/ee2d1b5a4921ba99196040264da989a9ce684dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/20334c94d11b2bbdaa8878a5d976aac5023003da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/28ce2e42979da2ea32c35f4b5b3be401338d4b32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/85ce92fc48dbd62600fb9e342b37921be6447a7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/9efe2b5dcdab06dd2cc9388f908dec5d4bb9f7a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/a23445dcb0161876203c24f90dfe35ee43760d21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/d90df3e6c232930215d75b0cf8bb72513e490cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171078.90/warc/3f66e59e7ec1fd264250202d8020802c6a513ac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171078.90/warc/70e74a426ff82e16a00bbcb1e69d938d0e4c2eb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171078.90/warc/7b8f4e93d47a00053e9fdfe69f0890048f9f4338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171078.90/warc/96e680d1ccb5167f6237bd7ee03276dcdeb4c888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171078.90/warc/c3c51d61b97c088cbec6ad548369dcad83267515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/6683cb7e9674dd179ffccca7f0f25a935570b924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/7d677ddf307b8c4867b444dcb22ca5ef2b608532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/b2de77142842796494dc75d870c67456c0551271.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/b581104fe2ecdc0203647616ed2ea79def113417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/bb90da90bacc112b1dae7ea424d221a248278d70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/d620243af7d90f89a8146eff7d6f248d4723ea1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171163.39/warc/0e838f18c2c68d8f1f5b01cba944723ec7ac2779.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171163.39/warc/2326ffb4fe2830511e087bbd41e41e3438f81c85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171163.39/warc/34379938b947e99010be00f2d7fde87d89136037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171163.39/warc/da35a33f0e4a905e69cb55a09657c1f7a370a886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171163.39/warc/ff558fc15375fd564f3257ae4f038b36460e74a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/1bb54de40c4840144d16d059907adb5b5d81b176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/21fbcbb3670a6efe16a4d33eb4321c2162d07886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/477998d39093d9d0eac8944064bc2f9ddbe06059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/5f08551b3d5fcc2917c4fb38e2515020fd01bfe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/c26222afc586aee47230c9c061f3b09dd7dfe88c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/f119b19e048ff9cb4617384d2e4a724969457f8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171171.24/warc/2e6dc1ba988bf30a6bc930769bd58a8649204538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171171.24/warc/417b2d8e1c94effc1e5db1ead77c499072a54065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171171.24/warc/9473b788414ec572f92f9cbed063f793ad599568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171171.24/warc/da91918e2a72b8ddc4e44b1888800da9dae7d3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171171.24/warc/fa0884438c81e85d8490d095408a72d280515436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/17250b78214736593d09067c7405a8e8d2c1c946.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/9ad6a86f5b5e1512c1c97eb0b78230c33fb50269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/ca0eaea9534c86322a173e5c5bab4332a0e4decd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/cdc70ed6ece70d06daad1cdd1078cafb1b71cc88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/cf917070c4f8f09ebe208ccb50d6fdc7cf21de17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/fb24a6d5e5fed65a054e5ce296e2430fca17c28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171209.83/warc/158e848410c3e301dc478e591fdb71820da897c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171209.83/warc/3294956098335e3fd58e6b64bc8a5c1d399cdf8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171209.83/warc/38f1974a0c321256a657e12a467e4f1fb0cb355c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171209.83/warc/be0a409ed71e10be9c78ca3a5c17136abbe747b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171209.83/warc/c64d28365e54d838d70990fdcdeaa008deb37707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/0081c9bd207bbcba25da28dfb238d124ba2752a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/93b65d431e2a0020251b2a44722a6feeef30c376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/af23b48066c83942bbf4e0fa22275bb72cba75e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/b7c036486ea4ca28fda5ac5f41d798cbb779a0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/e59c459ccf827e714778a720a2a0cdb5a90a1309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/e8db0ac7967fff59f5f7c13ac3e647c54857f35d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171251.27/warc/2fbd548c638c974063650e86b4702f9db01e60e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171251.27/warc/78af1d7b0df2a709a8ed7ac7bfa490ed36a3b90e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171251.27/warc/90cdb32070e6a77e6a6b689884fd16f54131be60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171251.27/warc/e5fd5fa96810900f47979b79ef4edcc19892e383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171251.27/warc/f86dd7b9b8e457a42b00c3c84ab2c3b5e9ccabe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/041c562c75c8d29fd8e9519f2be7106d9b0e13c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/6783d566c580d8d3d08da1ce50beea7110366760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/6f9cce2dab110d99878a190a00ca035a3a40a085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/88bc5217e2c631f2877f5fe84a8bef87ae4f25f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/ed4d2eaa4581516e10f6bf2abd24d798fb2de9ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/ff9b44f931e69b2c53ec409852afda16ad13a9c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/0296a76b5bc9280eea5f21c30aa7d8717623f8d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/2762c5190e9020d61b9035b5105593a89aa4c34f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/33b936a319a0a6c3eb05bfbf99629ac66e69223e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/d5cb25cf55b128041d7ea90bc9e723c929bf6cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/db7b1f0cad1b2dc6991da40286ec7b4d9ba92f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/ffdb56214e9dd3c1129ec81deb12f3c9cb5db998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/00980cf61baf111ecd21c69485408151aa9582f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/21d1b24a1d7f71a2f19dc51fee0d12ac570634fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/24c8e34ee8786a1eb7ab0eeee292f1ae9dbcaf6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/62ba5fda22e9a44e32c8e041145cd37cac90c919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/96218586b6288b36c569662a1a9cac51d7f1746b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/d4e5708fb9d4cf18e77055fccc544723066c9c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/1fe8307f27347766b32fb3685e6ca34c822306b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/27d32b1e5178bb7b841c8cc536ea444fc0879c0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/4d8a6dcac2c615536d52a86719e6bdadfabe8933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/c58165c0ba87f00804c63994d7613ff55569041b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/caf97098e3f4c5185b3228c7b41bc5853e13ce0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/d732543bdd884acd2b49c7ed8e2677746bd1376a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171463.89/warc/170641edd2f9c7532a3b38b71490762318eb549f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171463.89/warc/901c17ca13bc815ea2b4cc040eb6d515a8296307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171463.89/warc/92b1274c99b26ad525bfea22fd925a9768fff3e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171463.89/warc/bf060e998d0bcf802837181fddc30f4754445d53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171463.89/warc/c2ede3619f4932fa571248e17967b1f52075f6f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/051236449f81ba2694be8ee1d00aadf78dcce8ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/2aed144d5d87120d16ffc5028338be26b3f17894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/9ed2643163ad7774cd26854648daf6fe20b90b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/ac9afbf55776aca8020cd8aa0b3f12badd085f9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/b2dddc8f877d1cf4225bf4e326deb95bf2d21c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/b33a860bcca1b336e1acb7810082925e42380af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171620.77/warc/09184621fe21143b48222d71342e6a7586e723d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171620.77/warc/5459bfb9d1e645aea69d69b1c093956aa203b8a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171620.77/warc/7e80efd3e6e5eec4f3e1dda6d74fab88680cde5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171620.77/warc/a6179fdd971814087eba9a5088c6c132e951f8dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171620.77/warc/d88c8383807212008b1c31062d53ddaf28413d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/002572de4efa8805597efb92e6ce8e801146fcd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/0e34fd5dba672ad9e5a50f53c2837c09fd416ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/3bd071cd4019cc46d534a49729086e33ca86880f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/436f1720de1e5c4a058432621a50239398be34ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/6d9595171844b62503a54de94367f66345717c9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/ddb45bd15cb8f5a38dc60b33f25fbe5a07947a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.92/warc/058bc581af3b74622e8ab2c5c779dfa95da08989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.92/warc/1bb1c01f7eb44126bb6728620987b2b4f92b4036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.92/warc/48600275bac3bd2d299baa734bc12c2adfdc3213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.92/warc/c8d18ef1f602f8086075439429cce5eaee36a4b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.92/warc/e10449783552f298061076de71eb382356ed4fc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/77939acf0bf2b972434b4ba34170367c059f07e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/9556d1b4a8edcef6ed647dd4751ab6ddc5b16ab2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/97d7d27e302fb7ebb7c23434e71e91ccb6429b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/bc3e5955e1fae6f2a0af54d1e3683f1c15acaf35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/d7a80c1a814fb5bb03b33bf6b83c394d36955771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/f6ac94192c11ff8edc31b9c2640a8e0540767a92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171632.91/warc/2a93eb1ace44e15648008b14ee500ff5d8f17109.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171632.91/warc/80ab2570c9861ce0e358e6af3a6035e1fec22f7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171632.91/warc/8bc7936137335dab76e62d83373eaf6a4bc8a902.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171632.91/warc/d8a6b5e7720b3e16b117cf020da7c43c01c2a46f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171632.91/warc/f7b8eb3753b6ec012f81f59952df7b6b325261fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/1e82e59f5b23c0a2392824f8015fd0538c0980bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/5920a0ca3efa9697047c4cad51356d46f9880928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/7a8aba071b0b8180c707ed40f2d1b94f7f06db58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/7b8c2982374adb34ce90c8f431f65022c684cb67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/7fff019ff71e629bf3d739bc57c8ff1535be555a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/f351ced98d996d8ca7e41445f1b907e4f4dc5d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171664.76/warc/013fa9fae71c5da2feef20503b8b7e99420b4e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171664.76/warc/140d20dcb1afb03fd5ae0a13bf9735735a5393ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171664.76/warc/2198a14675e89da3393a21e55d8060c6416d54b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171664.76/warc/71c6ddd8added883eba45200815e1ff0ef39a169.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171664.76/warc/ac01e74481f735bbd26b70ae8d2e1ff69b8cb8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/0b5e6d2c8fc8c6be25c331c4027a0a8356ed2626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/21e49d16917602eb17bc8eb90a650d4de40aee78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/55b4be41f90952cb034de5181c1443e5d4e9c9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/84e8d7f10cd8b884c1e561ad50d0220b22f9e187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/ba416f4fd8126612891898f082fbe6ad11389e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/dfc41b9cf6f604e119d8275fe87bf5f9b40c4bf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171706.94/warc/32dc72056926294df9f242d8bc75d73b9ff08619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171706.94/warc/47b60431049fc6b47f0d7df7a83e758b7a9b341f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171706.94/warc/6f24dff854b8c2917e9edd8fce782a18e0c73df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171706.94/warc/79236f4c774682867008d8a0bbc1e2776031fe21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171706.94/warc/db10a4a34d24fa3835096166c7839e0a47dd9249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/10b25c35287cf7779f6b9b72fee5e437ae686554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/1ce2648466c755652a663a42a8057d6d60d2804f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/35c3c451a5a01e43966b6530b8c3f50e97b457cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/5d647288fe3f549238f60b8f0f9d0300dcde9fca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/6194e3a1a8be63a07ae4e32a49fe325466cf9482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/e7ad2d942c503452a0f79d1e998814c22b1a0c78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171775.73/warc/19e083716879f8b0f428736223f231565f807cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171775.73/warc/24f0457e2c172a185729c524df441687647a1301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171775.73/warc/d4482bbe292fbb825599e9699fde11b5437b6002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171775.73/warc/db7eb5a1260274519c4cbc1e45817abd7ade943e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171775.73/warc/dd7bfcd9e5a139aff632a1a221d5513afe4ff60d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/01af4c55754316de8eda990c77c0ccbc876449a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/50c72f9a2b3cca29ffa8111ac665282b0b509435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/d13ff840988e0d7e30f084b5be50c9e8c534259b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/ecc5bef9be4d8a2acaec038c2fc95883790cd609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/ed6a5743a7b732a48a597796de75c4e13028bef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/ee58f67d443ccb8cfba66a3af8d36ffa58154350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171807.25/warc/122efca0ea3d3d4b973f4fb3393cae596ae19b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171807.25/warc/12b14fdc5c911c59529795812775c0de232a7967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171807.25/warc/1c8ecadf9f7ab8c3045dcb924e16b3897c7080aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171807.25/warc/21c5de99bd176c72746ee977a6c78d272dab8fa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171807.25/warc/f8799cf63491b3af33e804925fa70e0d60b91a32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/2237ae295c66aed3899870d59ab8af1e5a0175a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/41ed5e2800e932c55314edb1a52754d658e5fe8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/763eccec14e5218915cbe1a45d82f8b272283f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/b8240456106296a309c1040a5b53f9ee6a6a7e3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/f6a420389892ea0cf89b8ac6451146ff419c785a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/f72974457902bdc6aeb0feb1b7b7b9bdd495d028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171900.13/warc/00d8892ac547d954e2b021156b44dc91429ae440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171900.13/warc/724f382b575da4035de2695a76c4f6272566ff11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171900.13/warc/8ea420300fc8183c06dbe593364387e0319496e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171900.13/warc/9d2a91e6f60cbfb2539e6c625354fa323dffeeac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171900.13/warc/e860dd8c451a8101219f57609f6800398dbf31ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/16ca8ae3307288e69c962d8b67c714bc1136bbf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/33aee1dd7f97a30f0f9655424c84bb6f54eadc95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/35aeae2e9326d4cc4c3df95cd5b2696fa3d4ac80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/578967a586f65e5439b44ba32f4ab2c8e2d4c04f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/a34b290d09bd0fe43cd6dff926d95ab7bade0ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/f1622be1a34ae6cace271a93b0eb0c0a69450658.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171933.81/warc/21e101b95434cf51c5c1b9651921e45f3cc78610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171933.81/warc/296d3687dd9a899495acabb7a1a94b4a2c377427.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171933.81/warc/7320b81b27d302b285955781e804e818448b9599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171933.81/warc/e2f85fbe7c5b0a84388ed69d8f73865e4124dfce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171933.81/warc/ebfe8270977116779b65ad9216ae7ccd761b6ef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/83b0059f7e6d112402e5761740ba473193b697bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/8b2bc596037e2161f0f5c9315200d3cbfaf7dcd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/8d2c3451a3b887d049abdb11696fdb9811d1916b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/a9c343b92fe58cba08e84026412dbfdd71be394a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/b01697fa2eb63f1d9fd827ef5196801609913926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/c96c47c99138ae5fa8f03e9ced25b5ee68ff3a22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.32/warc/5d32a5ce1bbe9e24dcfa6fe8c86be1256da0e92a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.32/warc/8918575062dd4510d099f4ea59f5e93f9a120d7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.32/warc/ba8473a61b938028ab6d1dfd27602dffebf0edc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.32/warc/c00081001077e314519e7795b870c02033fbf533.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.32/warc/d0b43a014bd32dec19838d554e5c18ce9c417685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/20bd937df514b31625b51e266c5d626fa286dc0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/6fb0bbcc0d7a18a703f6a261ec03d2fedb0f2248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/723a3ba27d9d016ca3cf846ff40e2c8611de0da8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/7885ad61d20332a96378031002eb28dd3c302977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/b510fffe8ccbf29c5ce16c913fd2b3654daccefe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/e24995d9cc843d123c391100b03a2d68ca7ade3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/495afb709cbab2efbaf366a3bf65fb5e522b59d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/7b7b88b34cdcb81d3a017bc3f1656adf279db389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/803a685732826c42242915d672c38f5d9b4f5706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/b77768762a1f0a31e35ee3343834b01524d23d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/d8dbf7781f4897e1c765c26553662056f4775bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/e58cf595f49cc3176fbec2c3b59719ec7c1740df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/18aedaa1481e4d3ed6495dac9755bffcd10f3b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/4390bb6d7a9b769e2dfbde358661345c689b1737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/443c3129b17ee836fc1b0bff3327fea95da58362.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/5fd9a88e8e7cda07c6ca5cd3fafa00643c7d9352.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/a291104a7266e4d26e4fb4f73b397ca3198c0543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/d9241e92c009c3f8cbacff92d06709ec2b1dc718.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/404c2ff7473eb4afcc3d2959a990fdbcec01a0b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/5a32d376b00fd6b867249715fa278956bb6649bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/716554732016b20f9ad74146f1cc3c1e93901f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/8cc6d7af7384f4b3a8c02939fa8f8c5b07a73aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/e278600d4bebec1b82e1f4d57dab91ff2752ba04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/f485345437a6ce1c49dd3e3efd5d235f9043e54d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172050.87/warc/3f61b78f468357ead9caa5b51ce27b5018d46e8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172050.87/warc/474ce2523a11c53d05d7e5580b551c91295fea72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172050.87/warc/489eef3ec6e436a0ebe2e6ac5475f875365c5c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172050.87/warc/6421ad1ecb5ced5d45c67b2eb4b5be47f0a78b00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172050.87/warc/fb7fd815d4a3c50dd15992e4f1d2d0ce01e53a5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/83fcfa9a8730dccf604b54d1c3fa1d413a6e4b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/890185ddc9c7c1e90910f6e230dff87d16d7dd25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/a37462b15724f758c01df7847628429d9796255a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/d6bc2c2fd5e5c3fb555741fc16dc99342135def6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/ed39ad8cb34a38e99d1c1fe26e27eb71c64b3a90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/f422e3788f458ed5b4457bd6263bd723add21d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172156.69/warc/2077476bb6035ccf158ab5d64917b0585d5e128f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172156.69/warc/2964b62025c29aef1c213e7d7e01bebcceac5d93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172156.69/warc/2e2011ba689331ddb7a27d27b33e0127789e2b96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172156.69/warc/6d078761ad1e5e356fb4a5a9a8e12bf7d817c35f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172156.69/warc/95d081937cdf0b7bfb21195eeb50ea0021f0da63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/03e9dd85daba410972a9951db83a39434e8d764c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/0ea4c71bced8deeadc8eb4c57171c08fdafeb7ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/25144ef0562e102b4f1fef37c87fe9baf685800d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/4e0cd9a63f9c673f0575fe0d804b06eddf8bfd8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/8c38fc21cdcc4f085aff5c4e2a5ad3814b5be913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/ff0c99ca968ce91fa2a9646fdc3825679d6c2829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172447.23/warc/0db097c426c5d4f29819579b89bafe1a4cf19d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172447.23/warc/214b0adc94d276c1c469911b3e9108235f3c382c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172447.23/warc/a0dc339f8ecd8817616f4ffade150420af2dcafd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172447.23/warc/c5bb089f39058c25ebd8b47b7f8a515332e22125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172447.23/warc/cae1ca077deeb1affad88c4f2bf4203fd081208e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/013c898c9cd3bd7e0868a84e9437492bdb3961c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/369a4c82f6994209e5b2e8bc82870620b83e4f60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/3be06514a15ba84e8f55c751551b2eea841af0de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/4b62174a98dd2bb5f73439baff94d06948272d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/7e93ada881a442f9081baa9c4389ddf05e017ba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/a075c0343ade0a3bf1e12b65ebc972e6b1574ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172775.56/warc/0d72407d18514898d8c1961377513bc5ac054625.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172775.56/warc/2db2a45248486a973688cefc730f8492fbaee117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172775.56/warc/4f4eb3629c5e00093bc29e48624cce4228185ae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172775.56/warc/a19aa25c764843c38f6ea9588c7668c51f0f2038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172775.56/warc/da42087b2d29983fb9cda3d066b49d93421a2ab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/0c89fe5c2d381a8c56852f706b43016c79dd2885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/a96dcfab3e71a0db53556d3ca9feaace29734cc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/b0e805351d3abb7b199cd32ac72d0c579ecec500.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/e13365bb7e52fc20bc1942d1d30627c0d9a411c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/e180e249bbd17bbffc2ae500d785efc11100c55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/e73de20132496392e5fb1f2c69e0e0339fdb5fb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172831.37/warc/0209f3331ac9e0cc63c82d56529332e6bb7897dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172831.37/warc/188be288fea07875c98d71cdfbb3f565b4927d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172831.37/warc/2dedc51959f47b649411bb5ba4c14b1cbf484e0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172831.37/warc/71c8c9ebee1679a0d602dd7e6330a2d728a59e6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172831.37/warc/89debd00a8677c86d66ff05c7d571ca23cdd2583.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/06d536cf764a4de1a0ca94d9ca0e72c1722c4491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/2d0b9e90954ab587d67696ba268c35d75bd747c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/3309d96bd40b37af2954940ffe3cce52e77023a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/620426455cec6c1847045ee183539785e7e93bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/69c565dc14e21c68d4615042f51a06d0de5800a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/6a1422d2bee04839160d5d8bfbbd657a34f489de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173405.40/warc/1547eb73a8a76a66a5259ea2fb8775a327ecc725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173405.40/warc/1f2f87018f7e12626142bb78bbeca6e530adb9d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173405.40/warc/2300e3a12052d4ab13ce69b24fb397b3db70fcf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173405.40/warc/7bfb464f2dbf56f08d967c86efbef7682303f5f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173405.40/warc/e949d7d6aef459958157a6decd91ec3ba72d1aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/03ff059bfca2a1dff2f6645b6f233cc5a4656068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/202da6711752239e6d59325a6f8166aecab648b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/4ae5b9b5d44a135628b17bccce6f848a8b2c1035.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/911bbc28bb1ed17f4542500e93f1d23a9975bf67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/da481a9175101c6a5fa3c5f6611b119e206a6a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/fef953749e1e7bf9e713039a73b49e0263a53942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173866.98/warc/05e7e1664d0ef81f565be1a2d59300c38e26cf2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173866.98/warc/18ebbf1b1c035a207e0cb127b1128847814efb83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173866.98/warc/49f3a1b1eea16946cab4b732add39b75776b44f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173866.98/warc/d59703c5ee833c75a9eb0bd90221393c8d6bf33a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173866.98/warc/f27b3bdb70bff8ff5e7a8724bea7c2ffeb7413dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/0d34f7aafc19e244eed14d87137eb649c7fec1d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/3e26e076785099d1f2da423f3429c3c6e70723ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/4090b26d98efa0ccaa0612fd6db56e3d648d2dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/de8ac58ae0386a99e56cbc1c8c69f483fd4972f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/e810e0458b98fe651225cf0db48545a8d9104d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/e9cc24d7b4b818d0accc065828084ab1cc247c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174124.9/warc/5a4869af3c816a8d1e240dd562ccb3c8e04f7df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174124.9/warc/7605506a88aeb145dbbd1e0ff76f8a9ac4098fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174124.9/warc/7f72fd01ce11c1ba8bcc9dcc3cdad01f583b5215.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174124.9/warc/b336a043c44692a1386dfcaf89419f1108bb775e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174124.9/warc/f912edc9a332ab87024de5632acbfb9fbe2aa13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/3fc7580e796e4d35ff800c00e3aef82005402491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/9e4a5a58c52b3113b0d6b9820514866c523f5d50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/bc7041969b6c335f2fefc8848ddfbf4cdca99c4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/f5c1629b5356f35f77aa3f9f09a53d2eb2ca942c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/fe1d26256c836433690851112d886867aa95a31d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/ff17918ca8fe70fa87751aeb0c0fda1b7e1b9233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174154.34/warc/2b57acf89badcaa7b3917b42f22fba23dd42b409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174154.34/warc/4c6c191c08e11815cd1f6efa8263fb5440d50cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174154.34/warc/7fd51cb8e75737fc2272b108aa5888c9bf28a350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174154.34/warc/87d90bf6f5519a5759b5cc5e19058fdeed4c1be3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174154.34/warc/f294fa18752ef1ba28895175473f11d5d0479c26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/0366de57b8fe3bf78efa13b3d17ed503bb723659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/2be412a3182a5b7aa7098b9d65a03ff40089ddfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/57d652be357ee2540a973acfd6e07ff8826d713d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/96ce81f68b88d432afdfb53655908c953805e93f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/a8b95f7367755558fcd51c7dbdc4d24d5ecab2ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/e9f1dee97631ccab7f9a449c0d26b10bebbadef5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174159.38/warc/4ba0ca0110986ee5204d15becf8707f7eb4a75af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174159.38/warc/8e94e7656f908511e900144c9234525c29b45c71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174159.38/warc/91c9da6cad1050356c4e45dab6342229c7fc5e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174159.38/warc/bb450172a4e1483636e3c981c1f901189426f7a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174159.38/warc/c2797e056a7993ea7921fac84c7f9e6851f8c011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/1b29d783e306313578c8f67d65a310965859e17d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/45b8b0ed6453a27d54e84f502078d1779605fb4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/5a9d7ee0e78a75db296ff62ad383fef27636087f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/805d2ee284a8a53222179c146b8c9c3098301b44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/9cfc750028f6a444c545405fac661c4f22a96f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/f942c214955ca5d206d6d4c45ef1e0f80fd5d52d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174167.41/warc/029ae8a79eb7dc7770fa477deffd4ecf75f2ab29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174167.41/warc/3cafd9493e073a5171f43abaa911434f05bc54b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174167.41/warc/60d1db74f75cbc971031f1d1ef23be4f86f1f4b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174167.41/warc/815e88e3d639a4a5046955ab8719e5aa2270e323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174167.41/warc/b9d18e88398e2cb42e825e1810dd340ca41fec4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/233db9aa163e578b64521d0c79f9e478fb93ea32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/3207c2bbdf8224fdac685662cbd44ec380c26145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/41adfabb024cb22e4e4483db9f11069bdc4bf823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/66a566ac293c1fe4755ff01cea47a4b8d2125432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/9dc081bb13003f9c840657c60c1924d21c2852ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/f221f66bbe0c674a8d1647b149f2b7f513eef242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/1a30068047a21f7c703801600920eed1cc83be5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/28086e1cb54b3abf1cd6d56bfc5540a5e519fcd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/ae58f2c7014bd9ab2f6146147504111f77686df9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/ca2b58aa8c006b63e0f0675dd41971018102baf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/e0ba5ca02c9f3f1d028a154f27845bbdd023cfbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/e32c3f80c7fb81ce518912d39dd2cc40a026ecaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1912fa9e59f6557fe34727bd24f2edc108fa16f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1b0ccd301d6b0fa1265227b1c79375a9c65de615.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/205912f5858a42d3cfc0bfdf57c3773c8ab6e748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/20747ef63a0cfcfd8b62bb917ec7d773f279d040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/237f736db511d403e57df0ab74d4d4835b125070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/245b81a4746801d3cd370848c089047b5a52ec35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/2a2cc1c4a0f3bd3c6e4a7dab4c756e9d2948841b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/2d482672b74475750650f163064c866c452195eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/33ee72f8ac61eabd7c55c853708c860e2509c22d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/351f86321cc6f684b8f415b9d71e80eca6135ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/35f5a7280e8a07aa042c425b63a5760f45a0058a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/3c047078b123567c598a659288e6dc1b22789d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/3cf04673feaee35524d9b321bbb0e3eec78372e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/3cf91c658c481ea85bc0b12e72f8300ed976a540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/3d0cd2770cc722bd5ac5c8638ad48e2e92e94df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/3d95cf7cbc7ef3e389e04d57e784b64aef97507b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/405a41bf5a5018aa758fc9608f9d211829c382cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/44fb2985c2c86492fa0583393c5496698e65c4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/45020a25c6cfc21aebe8806fd82d7103804f058a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/46f5538c801bbae79fbfade0be9f6fe857d60f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/47c1acb966ae3f7f8f2270de3e2e8dabd1b7d1f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/499dddefd28ebeca7c3168ea3d64db8cfed9e13a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/49d6a6a80ea547f6201fdbbe8398d0f697e1cd9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/4ecf96a00640ab5a2e6bcd48a7be73f6938ad03e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/4f60ad2cf3efff56cce717b2795ee138756e7e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/511b4059f810aeb2d8f8a571509b82774a8da506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/51d2480334f28a216e005b671d8f068529af617e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/524270a6fc8cc2aaced2234b49f113b8586b8082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/5271286faf361e5e51ce0fc6e54f8ceef959c02b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/59805694a5e99ebbc65746793e14669731a409ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/5b29210baac6d3a8e028f76c24595ed10db9760f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/5b8483b5cd9d1a2daa717632c0493e1c650cbf8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/5c40a013df8b72cfc9f3381d3486721e457c5b96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/67a0948e0b0f048ceca98154c7da13b0a167f65f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/68be68a4b837a8e4efac2636c7b40290da05bb7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6c4064b8486b99f9a1cf4946ca4b7fd79a88c3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6df8fb7ae9b66986df150c8a6838635c760641e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6efc7bf45054dcec05ea3f8175393d89929449f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6f2cd9558b0c0ba56a3eff13eb6229326661d55e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6fafc8c62d3ccbf71509837941a249f55c353f5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6fde37c84a7a749f3689997750def979c6f7a4a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/7099fd77edfecf2d181cee920be5f4fee75da7d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/729dcdf1c710fa3de690c5f1096557a431d44716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/7d15c3b767136f79a6805029866b6418f4829d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/7e636f15908d56b41df08c34cd4a37c4ead7178e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/81e3e57a6247de1a3fd292ec5a325a7001df5de6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/82022379cb10ec42772e69027931253cf7dc95af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/8464d7855acb6e79475dceffb4c8d844e139136f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/85058f047acdbbe8afe2116405c7fc3424c39a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/8887b12794f10c37b18a90270af99b390eedf7ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/8e8f2c9f42d2f9812f2d788a1f3fb50586e4db15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/90a353ca2f8a5ef681554666074daac2f708f308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/98cd34d7c4d5d313361562a1550deb15b6c930b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/99f969c081f917a154ead25a513a1dd30211f26b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/9d1280de8dd4d771cecdc0981226f5f043c28345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a14420df179c62d7bb2a5a3a398ba5362282daab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a2639a320b274ddf52d89bb50771c249cca7ddfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a3716da37ec3b0bd633614affdc85ab898728097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a3df3ceef75438d8c82b4923045aa0936875602c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a4d510ebc465f74e33d3bed9d591269e663b94b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a6209afe0d2616c59ac862ea1e25263778c6cd62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/aa4f7b095f3d05dda01689f3130597f5507e9afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/aaa3c3d8d85a7cb55b88f9d378d1d36d68c593c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/ac41b0c80eb6c15fa7d67b5b8dbe8b4edf18a919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/ae79d9eeb159a5d45ebc807424d0e140bfffb1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/b798dcac9a501e7a84b09397fe4d90122d02517c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/bc0475b84f0d19d5dd0e2b60a82e3717d454b479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/be136b4055a52eecd60309388c1280ec4e4af6b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/be6b4114ff094fa74dad20c836a7543a2fa7d7a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/c22194f954f44cd88d4ef0f44e49fbe67484b5d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/c3e19c72603280232c5f865e6f11b3eb10527b44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/cab83f5ea1f9425dcb78043fbdcdda875153e971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/cd39b502bb086068325d32ad7ca8e54ea0608125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/cfcd3db60a3511a87b51f1eeed46c04720333639.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/d1666b9ac4c04dbcb8ad4b8a04d68b36977fa610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/d242b8cc0f8e16735e75f24ee6dfab8b6fbcaac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/d5f01b5a9843603f1a0eb338bab4c34f65816837.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/db594525fb03a0ebf713a2c21257d9104cd9f865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/e2a50f2b2131546246e168f0e582c03184ad4b33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/e31d08d206f831c8bbaf15b5002def2f84d6b4c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/e4a28f975bdac3997c3ef6f190170836da27eea1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/e5d55700c79bc9a6d3c12c36ea143237765e5a6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/ef6dd2f3d74dcbdb0719d4cc0e0818a34de61b96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/efd9e6c8817ee5946757a37650ab91c4b3158bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/f17dd5142721b36624709a32489c9ea765e859c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/f1c99cbbf8fb80c5ff9337d28f635c80e8a5c948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/f46cff5a0520c2b27e5af57dc704c1ef014051a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/f5b4277918d4bcdd6af8db8704d9644b3edfcf83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/fa8fec4390cd78680b53065455de2e6d16df89ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/fae058736308e750763e771404c5651004d4bc14.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2017-09/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2017-09/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2017-09/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2017-13.yaml b/configs/cccc/dedupe-url/CC-MAIN-2017-13.yaml new file mode 100644 index 00000000..825b2657 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2017-13.yaml @@ -0,0 +1,686 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/031540f05aaf8b728b97ccc6bbe1df76b62d6094.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/0bef1ab7946f6100359bc5f947682383aa803e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/0e046993812030bbac27ca3a9e392dd806497ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/118718af885c17b26bd0a4bee318a97deec518c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/155902df16e8d7c0f50466b6d643102984baf4bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/31624fad2c942d8079b8a419045d396628838b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/6f74461800a87dc510501fc937487ff4a7646712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/6fc80f83ebcd38953a46df6ffba8db2da0a3e516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/95283baf771ea1ca989e30085469c5ef9ed0c8b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/f14593547d35d33d968f6f57e0666f5e12b7a435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/3887d7d3956becb21eabffe6281d14839a4cae45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/5bfd303a883152645578dd7e37f499094b446746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/720b857ce87ab49ae21095704d3e90aa16bae2fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/8d097370a5135c4190b23ab611c36b0c23129c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/afd931745c8b731f156a6cf2a642de95559b23e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/c09f2208be7ccbd3018d050a3dd18e97028650fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186608.9/warc/b10127aa44fdf41f4c3be3652fd5258751f7782f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186608.9/warc/d5a6eece8d284c08d8b68d4a7610284970cb4957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186608.9/warc/d91ced3c37ca398ac6de364bcd79e71d3543af61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186608.9/warc/e31996816943eaeaeea8719e26241fc08a6dd1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186608.9/warc/ebca7871fb1aeaae2cc365beefd77a8ebb9f66c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/377324d0d81a24c6111ae958590a0e7af9b17c1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/6f1e015ab4fe1b3b2f06010e23e98ee961d6268c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/946b64cb90719ecb48413b91085897fa4c0d3a9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/bf758bf784060c64eb4f5fe492e8766577e6705f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/ef8b1933d189a3783f5ce910dc89837a0485986d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/f725520c1fea1f54ebe7e870000f0ae3b58f6a0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/112a4894f7aeb7de547bb8d5e26d79e6f90707ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/1b702b5f1b0889f0b43d6209ad6f7e904c74b9e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/2195bb9a893befabecd96ed7d7096dbf530a02a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/3f9e2aceeb8e8dd0e9008cfdc6f109b5f73ae5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/5efc2e06c7f0c6114264db1f1fc9f0c3e3ae22cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/674a4968059cea33bfe92e6205f1fa744c5adf93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186841.66/warc/41bd5b1f476ead01bd1c1e4ed23f936c0f5edb39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186841.66/warc/481893980173c69679528093f80c3aab85ea7761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186841.66/warc/55d052dbd2fb33c856e4291dabb86b9ab60aaa4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186841.66/warc/b7921840a99f83e786ddc2f8a9d714da2d2e3904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186841.66/warc/f8aaf01a59b7dd2cba2bd5faea0cebfb57572e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/2c6ce289b1861f1659f5ae48f99e0d5414450011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/65aa742fe0148e04b521da28920b1805dd0b5058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/778541e6f483d6b94377cff8126a1434d1bd9303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/cd5363ae8d6db02e0f26ebed83a5088c30d0b86a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/dc641ac57e64c80c6ea2cc2522075ee2864d7f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/ee3d404d96e5970a0886e91d8bb3c4b36040f18e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/20a22e987e7edaf3de79365e1444e14664cd0807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/665bd94432c6fbb97193e3f29c98e97e21705e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/b0c9507ab6695cdba516a3bd01c64926e78f6597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/b6756c04cc59b4d591eed72725b2296558c029a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/f784337fdc776f2a45fd8f4281fa226b36f34a5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/fb8a48f149952a16168bc72656d703c6c1406273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187113.46/warc/6a35230536d00e67d491777405d25c86b67c3b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187113.46/warc/6b25277aeb05630a028736d062eba61556c58c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187113.46/warc/7b827b65292b3fe149bf6a05fdb564e862eb0906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187113.46/warc/cd5b428493319263d99adefcf374dbaba24321cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187113.46/warc/d86860c1d7fd6783f97b7fcc093bcc1f566a5027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/37e4033e73a97fd2224bd50aef419539597aa339.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/3cb91d2aee2ca6cd5992e6554e17812acc75b3aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/449d075e262ad1d86509ead5d9f24355a135278f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/5c6c4b3dbd7538b9ab66eb9cad84cee213f3f2c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/74a56c27872679774eb0079284c4521e8e787631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/aaff03f3e6906e8e27e65344da55d412a8320856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/1da6833db617a986774b01cd7f6c61a536069f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/31c4258ee44eecbd9280be7d5f75811a8278b20c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/679d3cbb0b23cdf40837a2a9ad958edbb7c49049.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/8c95132fb708ebd1880d7927d13ea465c187893d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/de30597ce83c8fce490fb3a551dc9f507c1862b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/febcf396e189a71644bffea539479081c8cb85f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187206.64/warc/1fd30be167d89253de8a1009cbd0fc550b0862ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187206.64/warc/647478d8f1da0c3b7f08736a029a01ad1c19b56c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187206.64/warc/6ed6da1adc15f369a59c7f5fa93e2cf137ccccce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187206.64/warc/85ac3d5b08336881d636ee1eb83e5b43b9ac6182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187206.64/warc/9c3e6fa0b17e89d0ee6eedf138f3f6cd8ece067f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/12e65806529ba4968dfa0687eb52a348574d59ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/27c96fe11fd54323b98e1b94f91241b758069b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/4a48f86b07d2091fa9e9f7954624ad43cc7018cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/9935a1f31f16d2c78d51db62075f82954749dab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/bf7de6fd4f93022cf55d86d3812a6bec10f2a5ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/da2ebf9148481779529dcd7fb127ab8c77e96c10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/030e2e4c2b642d8bfb4cf43e828b3b2283992b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/77ecef05e65eb202aae561b1f7e1b8f4aeffa38d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/bd1011d090448442ee9f0c9b659e19f110be3d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/c0d2b0f24de504fad837fb9e46b6087d18d05824.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/dff068e3a45af93281e33389263f9c7ebb99117d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/fc0868041378742db7bf522cef76c92b139e0cad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187519.8/warc/43b1ae2be1d355b6414488e4ae8341abec7b07b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187519.8/warc/8586579e5866d1426f12b4f97ab480ad6541c130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187519.8/warc/ab74234149147553ddbabbebc5734b7179b0c190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187519.8/warc/b183b0e42e88e702f3dc6955a7d979759a3d01a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187519.8/warc/ea3fe24976c78b677c6a079c0bd205a8a86199e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/1e99d4fb824545c6820de72109d110bfc8101d93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/5164a21cc36bb8e0f185c61eba59dc0d646b202f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/768631222eb3ec9839e386e00ae73a63ac096a10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/8209ed1e2dec856c810340a46afa915f187be76d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/8e696082b6fe981d3da4f0bb223f73cd707f56d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/e569555b3c8ab871a4d482dc818827b5c4c8a4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/0acd4874a5f6e76313afb8b061e2d9ae2cb6a5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/293b128d15f16f2d54be7639b4c6e6358c5f27ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/2e761e53e36a20a556188ea68173525720951d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/31868ee0fbfeb0af286076ac3b631c5661e38e6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/830121574b8ebf6bec5b110331a07a7ba2bca5cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/891e73f4efc7f8437b94f6f314b92d5f00043e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187744.59/warc/804d8e2e4c3050812329acbc1116c7c34185a0aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187744.59/warc/88f7565019c399215a76fce277afa23f5c616d54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187744.59/warc/a8fda961ae67bbeba210898035b10d36fc022367.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187744.59/warc/acfbfbcbc7f61fd371d8dffbe40b11e5ffe72220.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187744.59/warc/f1afb2b8a20c30d4807be8029c0564c60664088c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/4c049a55ba22953379bab835087518429d610fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/5152b51feed5631839ab31e4eda984163b296a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/9a7dff0fcbd8495fd94ff0b6e8f0e230f217f32d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/c2b0b4839f003152bd0b6f9b51080cfecc4c573e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/e21e9dae8c181360e6ff1ec1c431fdf1abee758a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/f824f08aff43f6959ca0c6a35ba481a0a56a20fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/66472b6c471d7040226af93e4a30086fd9127cc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/8977b557afce521196b68fe3c49dbb542833be60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/ddff6b0dc7895ece8a58ea283cd1729112096e5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/fc9bfcef6841121f2a984a9eebad0e3fbec6982a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/fef9153d6682d5dffd8506f79156234904d3db2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/ff23b46103377c43128f6e0cda3c5a8eb60aa1e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/2fdd676d0c15382ef9a7d48dcad905e8dbc439a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/3a8f3445a3092342107cfd7e98054ca42b20971e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/7124a9b186258cf62edb5c13d40a96236dbb6b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/8231db430ba99e58a836fc91f149d854ae5c094b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/915b4c57dcfbaf5f4708c368e5979977dc3f3e48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/bea4793a914e79e7c1e713422601aff881511b75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/1c8a0ae640a32b2064d72c1ee7b0bc9470cde94f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/307100dd9047d78ac5f5a43e8c3865e2f1a551eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/a64250ab97573892199cabe46946b5d25181b36b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/d96c3d6098d3a44e49bf1eadcc603eed93b4957b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/d9d8d763676393c3a41d9d0f360c887a4236a4b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/ebdf721f9687c5365cf87f06706c8ff8f70a95ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188550.58/warc/1672115f8efb3b9faac361ff21197f1b6e9727d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188550.58/warc/8477e4185cfe04e1c97189f2fb7fa6670fb45943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188550.58/warc/92f3f215e1aabd038efb25c5e4d3c38ed0444a2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188550.58/warc/957af685935d8ee0d6f33d7bac54f258cfeb72f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188550.58/warc/ff1ccca8e390acf7dc450cc26adb25f18df579c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/0ae3c49f3b5f086bcf6ab45cd5186ebf9101985a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/1ef2b4aad234ea1829c9368889367478d1225d38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/3e741b12cfc5284c190b5d793d7fcd3bd32a5072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/6b4e9cece55962a476fada5243e2fedd1d02565f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/7ff33c79f88df8414e19090362e19f25faa093aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/b3a9c3f7bd7bec80d658f15e7033938cf107d954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/1f96c0bb5ea60896a8a409b43d394aeeafeb53c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/4d09655eb6a7eee538731c9025be6e284707f97b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/4e8748f4c50971f2d3ddbd0ef5af28d02bb2710c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/934bb1df34074d94530b2b7b0331758d1d504917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/baeb3b16ca3d59b15ab73670b8eaecebf1676e43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/c662ad5f611e5a3c3c60ab8b3a6a99bccca8b589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188717.24/warc/162dca2b9236998deccaffe28c8675ba4b6f5912.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188717.24/warc/9c29d33f9a736e94e4961411bcb92d4da1588ded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188717.24/warc/9d93a7611df89c48262976a307b9b29ec744eef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188717.24/warc/cc7badc98de2ea0f4b6238eb8f7c0d4cd7948b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188717.24/warc/ece6129e89f6fa4ca2578a3e226eba1569efb5c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/11f41a0f93336201ee6782c592a127d0eeb27018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/38ec71d79923f8cbb69b0a46957210e436063564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/4986989468b3c52b9f6e187ce335d4e0c67c10ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/555629ea79f2ac90974156b962522e92e750e184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/aaaa47733ae4eae5ad0410d6b7529cc22a062df9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/bb191bc1377af343a735658712e35ac90f1f1aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/140ae2f47aaaa8f870947fd8d55c65629436e5c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/35ade474aadb8adb37b05f498569b0350ddca366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/862272e201ff9bd1e26021dd4181717d3d248693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/8e947389398e256957f88b26621e5f0eddba1ce1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/8ffc7e8620af6e9bebc5f28a1d81032450af87d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/9669c034bc37f371738fd29c30f312efd851c86d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188824.36/warc/00bb049cba1f32167bba3aa6092814929848be4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188824.36/warc/3c627b54335d7fa2a1fa98b3f8dc01dc2f89332d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188824.36/warc/959cc49ad665b17b081199b88c03d015a88b360d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188824.36/warc/a17a9f7a2f2bb47db07396e9f50676a71a18c201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188824.36/warc/f5dcc3978e9bf641c2df4325222cac26ed77a9ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/353d8d4925cf19d46327749033f8e012c2a9b5e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/6a0381e2a8959772bd656b9fda87c2f85041fb98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/6a7396f9eea87ac3cc088ba8aeddc4b1f239b592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/b4c0737ec6a42e90008bb7076f0cfec941d8778f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/dde0e9d8969209d164a55dcacdcfce356613bb09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/fc4eb985c48a629a3230504ad824f7171426d1a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/091e52477a8c5e6df7576a8e470cf42b45535627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/1212bc9dfc5155a28179d89af562492b7fcb10fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/1e8ecd61656ef6a1ce8583626502577ddef5dc01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/2038b81ba8f5b4a96199c9a265552a9f6aff6769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/40def3b07dfc85f1d4ac825bbea56afb72a1613d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/ac22a3dd182704e5d69b3c60ab7211d0b0643963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188924.7/warc/15fb70b46faffc7b44429657467f1fe8916e6501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188924.7/warc/713aa5f9dd3c61df180b2ce2d2e27c09b66627cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188924.7/warc/9a7264350989ca8b5d5cbc17aa46a8df545cc3be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188924.7/warc/c51594c0ad7386591dc34dc68be70da7b645556c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188924.7/warc/fd3a0d0589fae6bf4ae289c24e5766f01adecebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/0d38f3403b999e2c3684eb6aa08c6a64dbe4d55d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/186f2a79c5aa978ea178afc3ff0a037b195bc163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/49e92f3ad55818fcd4f73a2af8dcaacde396904f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/6a1848d6f8b4bc5f3a9dfc9a1afbb72486aec391.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/a6099fb68647a6becfb1eba48a18d398023b6fb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/fc5318781a805c5cbccdcc61df2684374a03a9b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/2c22c9092de3ffd77105d0f869123f5003d70614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/7af416ad24b8bc6e92164f23e35e89f357acaf1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/7dfd17b5f84fba77dec743d823c56f01969ae7d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/8a1e618efb5761a41f1a785a8d789d8dd27f5c44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/df5d63a455bb3a5337e5c2d3977a8af0cbcd657b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/f158c5bbff46b97a5d71a0da5870f47c9d11ad58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189031.88/warc/20d70cecc06989f405eb22cb8aaf0bd79fbc39ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189031.88/warc/763e6b07be08e557272040ba8e2ac7e89afe740c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189031.88/warc/78bbd11027fe46b086342e03aad2349e4667934b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189031.88/warc/83309b18b825a38e2b03b04b7ecf03d18e3ddfb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189031.88/warc/a3959614ecaa1793ef66cb1298e35200d23b9e81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/4631e980068c35d6999e1e0b9eaeafab92d270b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/5d5bb14550d6acc66314cc821ca43b560bd64376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/8aae8f5621268196ab3f6d78dce7e3b35b13f660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/af90ab2517749146c67448c97134395cc235fcf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/e65fc0cdbdf2d36a881e57dcccd543702d54499e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/e94b533d917c2930641783ea952f351092340072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/2a73581c67adb0ebf427957b03ee6b1f5d1d2a87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/2c82872ea31f43822661ac371dce460e6713aa5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/4717a68d5bdd15ac22c5b6a1155009b0f36f0ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/acb14d205727fa4bf4892637970757ed5c9b5648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/c52e343764efe1dbb7f15ca2383d099c9ecda063.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/d455d8de44fe328083af24cd78a1245fce6630d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189088.29/warc/2cecc10211cb97602bb63c8f00c90dbd054382fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189088.29/warc/43f580d87e9882edc3fa02a2fb3f306a52dd5ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189088.29/warc/55c7dfc56f9b666cd8355d1be291e19750aaf2a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189088.29/warc/c248cef7ae141e54291118d6f866270eaf4162cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189088.29/warc/caeac8b5312f1c2df45515a3a3c0125a332e95e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/211f056d1ff1817e7484f7dcb40c12569e97f27b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/3279eb7ceee21775dc9fa9a54c1e4d801c91ee33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/333c8373f6a56612a0c8bb9b287728edd4db6c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/660ae1f3954ca0bacd421224a4d62cc055369379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/b13f95a2b74b8fd5fa9e4be1575231ee284a6a69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/e17964216a1f41262ca59e62a051437d5cdb4e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/0de35c15e14370459127fbe372c677f8564c7e24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/166362abc079b2162e77da4f25e872b2a3cffb88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/8bdefe9349bcddfe8e2d6c61c8a8b4570c0a7347.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/bb908359650a76e3f5a996f07d15a4a79304b692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/d55f17ab6eea09a75314a550d785f04780053e2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/e330372642fed2d0bd63a7c7918dd97452314b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/433563c5ff6b4c77476dc26e6fa94134a4e0cbb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/63a2d664ff2633afeba6c241b5a5c0283f67c861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/8967d108fd9eb27ba82414eabcd36b1a7dab8b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/9a5aab0c322a19a5ae20e467a90510f90e6f1a51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/e2930e4b0f2978e04b96c306c429963fa691e941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/f66674c98050958c473cd2dc1c5549a96baf592a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/16abc540e9ec2033f12e29d4f9457bd7750b9f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/1d4e944ddc0acd7a59b14e02abf2d158fc356d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/2d3c3bee6bf6d6bf249bda4dc2b9d5444f796bd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/2f0ed187f48a04f954ed28ec61bd3cd09fa59439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/84ccca6af6690f6184e0fe4da514cced22ed6f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/dc665f2f2ec76c9fcb0cd4f0533a62d64d57e8ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189198.71/warc/033287f5ae8b5ad05947f579e8463e1160986d9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189198.71/warc/045d6aaa4c87c40f53ea96dc547e77aa6eb19d00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189198.71/warc/30577079f7c5aa6078b35a302232428f6bbef248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189198.71/warc/a26052ce867a720588f6f998caeb37e496439a3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189198.71/warc/fb611a00ec8f67bfd4ccd4a07771000d0e4e20b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/330416fb463056d173e5a3d42270e5c216a3962c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/479306f74d526a7f4c6e220db0a439c35b22424a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/683caeb96dbd9c22361ee14106fa9e9f6f692822.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/a43813d78368bf8c8a84e9743a39cabef044cba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/b9e043afe1b2d4997dbc5aa2a057d52c5c91bf0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/d3c2c25c91023e55a1ad61225bbafdf925151611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/20d3acad869ca17e735ab8bdb85cfbd2b19ce8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/27fd93cfd0c05936dc0a44f5cb16401c6f722cd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/6c51184776a5629e2d8faf501f256393af0b706a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/b26c21cfd5be9871b6b340398b5f10e938f6076b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/f234f957c83075c8a5d2ac2282c5bf1738ecdf0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/fd2abd0bc6e011b6553b67826bb6f87b57bb8a07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189242.54/warc/3562e9bb04f9d9ef2ca5554b00fd8547c2c66367.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189242.54/warc/cbf5f0d8972e9c8c81fd6ee6e0c37d09cb069fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189242.54/warc/d9289a1063ef5d3f9adec94bdbb70d75764e5018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189242.54/warc/dcaa400db07d1805d4b30e029a8ff79cc48b4f93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189242.54/warc/dd8b903e07e8ef4f62235e2c4d5f8db4c24ae09d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/18c2fb3f93b8b4e6b60f3260cf2d067ae7308205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/3355c7480922efd18186d18db4b515cfaa5d0459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/4d35e62bd75c1316d4784d39e85b845b60aa9091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/90824885c1b596bc18b44861edc2709556921b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/950ed084a68b5a912f0ba769896f03aae4ac6f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/fd98a358490c9c73466dbf6443e1236f2f76c3c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/2d65321b6f011bce6b73e4a16e6885dc58a6cc79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/42c76b0992e6b1300a11c86a65bc1f9f8f42c620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/5dea8d074ccbd725ee720395263a32563eed8d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/c674b1484c8e172ca68f252b4158ffef2c0ec9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/dd0e7caf0b3e214ea485b0cd0407a3ad284c76d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/f3db4c071d81e37348f78c61efb6e248e5ed58d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189252.15/warc/3fd067d5f885005f22ed38a685d2e6f6cdcda9b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189252.15/warc/7246b7463bdde9bd46183f830faacb4a0278e651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189252.15/warc/b1dd7d13f0d944ba2152c76a10616f5c301e92e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189252.15/warc/cc9a67805e9936cdc3457d720da5fb30688d2009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189252.15/warc/d42a641cb869e9a3113e88231082399529298563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/02840cdc9c86f61a9ec3993c467a4e004c4c6132.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/0c3b3c984cd2bc25cf501accdd11a428ed6a1b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/1fe888dd7d80abc15cd98e159d37bee039af7ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/4341fb892d42c10c983590532f67e31069c6719e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/b87aa5062a6e731652371b8bb2d50305e3ef6169.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/c667e11ad258d2e57eb712663fdbc07a99e81342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/a0757ed2acc1f0c6e0b94b80dfbcba3b060ddeb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/b4551ea11fb4ef2d82f3a0e908d9a364a7ac256a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/bf65f4762d391088fd457de4548b7d89a7b00f3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/cd1843b90b5c8ff88db52366787c98dd1ad40053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/e31851227469851b0d72c3d4b0ad025de2e33b25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/f165a2c046967992384b744f68e8fc11062e30d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189377.63/warc/014d53575c7eec807c61af9a65acf3ef1a5ba71b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189377.63/warc/5d19814d87ba4239ab8d3f59428087b622cd7005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189377.63/warc/af90639bfb289bca0cc434ad2ebed78a652962a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189377.63/warc/c8458f78c0fabd841aac372bf14f2e1e119cea61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189377.63/warc/d28b81cae6ca2ef21240b34c337f778fb2fad474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/1e7797f6a4df2c849376604b8c49b4f00c9841ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/7e98d8b1c57c4621fd377505b9ea25135e97cc93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/bd95e110c8992fa4a1c7905397bc018416aa3104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/e6938b26eb8ae6c250df1a37d79d5e19c7a888c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/f2b58fc8bdc51f0da38febfe9814d1fd2c0cfad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/fe492894bf43f191a2d399d0e75de341507e6f75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/0346382fc4ea5e6ccb6635a7e8e780470137a222.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/3225e22aa1ea383307dd57966e44c484ec44c1c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/5d14f0006725d7f2eefe92fe734f1478e292b22d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/b0d4a72478269b7b4b93fc6c9abedd8481582a69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/b2958877f3576c7ac1af30622da19d67835b684e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/bdb0dff0d3df6782f33a832d9c19374aae3a6f8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189466.30/warc/1fc51ba15fee95afe3bea2c1f74fade649a62b17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189466.30/warc/50f676948f1e679a00a37e5871bec71cceab36a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189466.30/warc/525531c20f5420f4c43261babb3c4e5bd585f890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189466.30/warc/8ffc2939ba3ef4a3d99afa919d77113d28ee4b84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189466.30/warc/9116fd2d2328172f099986de57365f17958bdba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/0f188ab924416b026dd567481a26660234ab1113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/2101fa71b8ca3e5471b2a2ebbf2f2a1fdad03c43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/4c48629e29b061b041b6148c17c2f96a76437473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/6078222552824ecc0afc34c5ef1424ac3e9af20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/6c2894ec897e0b2c4b0eaf1d097ea42a7bfc18ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/f810c8b283ed8739bf7380196ec7193a517cd580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/1f8abfbb880d9faf830e9e7de2a879f7c85c07b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/2099b545bff9c419dddfcd476e40f7ca491aedc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/2401980b826beea2d8e6aa2d55afcb2df472cffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/4324b43730a042242485ada8293df2a376c7d995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/608ee5a970954bd2456d27b2bff4e8c01d9bea2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/f49eec844b06395bed177dee519090536aece909.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189474.87/warc/1524b7dc8968f73338d8290e93ef3eb0bfff8985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189474.87/warc/1ca9834e4f3dbb3ab784d4442b2a84715c1a1bf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189474.87/warc/6ea14bd7f61b6adc3fd5df340d9e4a0733afefc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189474.87/warc/ab4f1bfe5bda110c1d16611d534a86f845126716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189474.87/warc/db645af4672f1dc1cfee9985ae61e572c7cefd43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/77e68888e8c7beba51f130047e761cfb1e0c5989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/89dc0890cfce82fd17599c07f513bd6a34465bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/97528e41950f95bc0cd7b1e5b7f341ee19e9f9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/b890b7ddd9ec318757531a03e0a5f8f1b9fc4f68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/e50546b714c2d3c12515230f9fbcd38bf5b655a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/e83d98b8eba374d2bcc6337bfb575a9760671a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/10c751065914cc27d914c4bd0b65ee60aa1b1b50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/2e172fffa38972eb1dc85a830d4e4b40eda59b3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/312385cca475bda552ecc5c8dd2eeca4810bde18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/b17b0ebb8477ff8785a36c625e33f29d0832d899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/bca08bd9d60bf4bbf2cd56e7dc423ecc873ee6ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/ee1edc7101382b74b896ab37898b2eb9d0c34715.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/1c83d994f15689ebf35f23190ab7f7447dbe3d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/3187b9ab7fdc08010b2e266b1178316036d221e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/6e057009a87d832b406948c85aefb660d174cd49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/8c44a04ab061b57e131d9482286dc39ef995cae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/bc9c1fcbf94c13eb382d48e57863d5cc1706019c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/d0c93c64a93e4e1409a89328f840f67fba3cae91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/42fb587663711a3e10dfaa58e7ae602f69c35bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/677841307a523c156e8e27022ff947998ac7e0a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/786f768fb34ee7802470a29d786e17238f5d2927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/83e03bef37eb43d462aa38f24b12f56277b5f60b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/86fce61c1f0dced3e26ed0a0dad15e868dcaa3a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/a6ee8c7f4b5035a7a46687d651c4cec294fb0e9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189583.91/warc/02a7e84abe98ad8c87b33380d6c1da7b9e6f26e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189583.91/warc/6166e7c10590d5f8bf6b5665c851e38af7eabff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189583.91/warc/6e3316d736ca28c8b6f689306b99c5f5a65571f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189583.91/warc/a013f453e297d53dab96928ed815e692e82719aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189583.91/warc/e2db8f96655f440d70afe2bebb846d504d61c6fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/5c18ec03ed4b0ad21c30c19f7d5001ed187de535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/63014f2207ccf964d3612cd7747efc5c99b57817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/69fd4798081c2416ba1a9eed436e5ae36f230f9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/96a17ba96382dd9ed0dc06e7741f9571d51bb8b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/b641701ced5557e772b385c8fbcada421a152271.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/e3b2500849addd26d6a17e156497719675d0ca5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/640330f38f79a093179d264472440e4bff79d33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/7003317d3519dc65cc47cdce1f2fbb86e1e897f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/7129d6ba469cd615ae4162f88bbb1747dfc95c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/8ae57e8d7d7d7fb8022374c3b659c733af0e5ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/d77ce850ad0a5ffe3a5cbfc59d642518e32fe513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/fe4b4e4ad933fa5bdae992f9411f6b0749c3b2d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189680.78/warc/0aefbd8fb6e941f584b3f2563824b55063b9094e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189680.78/warc/1d4337ca9dbed728344ce58d9cc297564848c83c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189680.78/warc/28672ba9e3b4e35ba994f98fdee4a499d48be73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189680.78/warc/5e647ef20c30f4d3e5947e89da4707b2d026bab5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189680.78/warc/ea4d8d2dbb6936dbdbc9993fa756b4efce4d0571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/2120704ceff338ca5ca888b94269aa0e7087af22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/2139799e7bf1b475cd23ce87e9f2222d7b916f04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/324de28b8170cd6dd53cf6a41f655304f6f454aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/5a2d4bd96d3f245a96e7033c11544fb831384e7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/bf861a7376d6992c7233e2200676357e511623ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/e8945ba7d7539cd48febf4789eb2f90f4c2ab8bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/3ffe2c396c5657a93455dd209a6dd460ba26af5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/8c7faadae06be7940b251633ccc35412a2717581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/9fa34a45e46c61b015b6be0b85bc5999b6cb3cdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/abd6b5e76846c0f29fb63cbfa9120884974278af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/ad789e3fd84303a67d7206b87361e1eae9705053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/b6596334936b65394d2fb6df2b34c9cc50ccd68d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189734.17/warc/01a92dae933afb4abc7c897dfade820d041c78fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189734.17/warc/5e28b3879b0eb139f7b0f4fa9fec956d66ae612c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189734.17/warc/8246a809a7e67ac58c97f065bce28970795dc0b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189734.17/warc/8f0b080ddfc0c207ba8105abdbc15058f64670e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189734.17/warc/d86c886204647d2d9cb393806b345aabeb5b3979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/20584ca90c7bd202a3ccb3298a5d0541dc8120fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/2c140b27b1072a3cb1e15235b7a1c4c88980fe83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/5ac3afc8fe71d7db7463d18af21a870fe873e9f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/b385eb8c8c0351a2be478871f775f4a665914307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/e4ff7fa9e7900084f95f9a5a2ac4403f2fe232e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/ebc839756ab7703ae68070c9677acb582a9fb1f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/3d6fc1a6c230d11f7ed9f154d946ed5e642f1d51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/64eb163c65ffb992a70984a92e5dabe6b2940e29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/a8029d5ed482483799e1fe3603d174944995e9b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/c2e6e6e9ceecebf4838c4625cc663c34b199f464.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/c35787a44f3d4e814b9378e0da0c72ce6f7e2fe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/e2f413bc62e20fabe9bbede73e15080ef36df5ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.72/warc/061a6c5a8b0a771945f239409a71e8c816897fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.72/warc/1354f7ca9e7523616066fcd94493d0aaeab9d5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.72/warc/16b5ab0d98bb929e5dd846aeee7ce162f58c7da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.72/warc/d8a9a1daf2aef75281addc1c314b05be9d7401d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.72/warc/fb249bc95a54c31b60a085ca218d815e9523b787.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/2d0ab37ef8cef100fa695889e10f50e1fe9f2b7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/7121dfe1c44d7e63b5ea83917aef01de3e7c36c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/8f4a476573501eadfaa2b168fdfeaa863e62c6cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/acaccf111d1639ec28f88a22ea10a4f0b48c421d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/e7dc7e295003e54ce5cc06aa9e65282d731b2a6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/f88da03bd2f4a2e0772adc1306ba70c1d3c5ee56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/17d96e9d12cb4975b3081994405687dff6ecd0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/2c34a81316ead1a0726cc3be805bdf051853834f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/2c8627464fe2e6e82deee3c2f2510157bc655246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/4fc60e58ff0ea1874aa1975b7d4fb00076c41faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/82041f3aecfec7c65526d23fae0bb9331a34ae47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/8a2f913eef318abb5f4b0a97bdecfc11149a561c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.25/warc/0c5bc144b76e2a87c10a9d1cc1971c905f6f4605.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.25/warc/1f3861210c27c4ebaded021eeba153a263cfe338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.25/warc/291acb746cf2e459411a37867cd7e745f2edf3b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.25/warc/2c03317cd8339cf47cb2410a9f433fcad6d82856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.25/warc/48f169e0e696d1b9ec0010d6ab29b604e8f7b13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/55021e7677ebad6c5dc5e996697a1ee6857b4c7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/5cf685ea796ea5763e5ce34fbbd0cee189326af8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/adb83067e911a5efcf956da4dad95c4b2604d856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/c2478e28cfc432664e6929cb5a3d2f0eafb5044f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/fc03529d7ac57c9543bccb2e7d344922e1e0d675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/fcf197bc6dc0ced86081422af74dbfe6e2a76491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/0dfa3478a8a684025f9751b1358ffd599a1f6cf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/1b2296083fce746095af5c2536584f96d7bd08f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/370a8b5355a24123a05464ea5c244ed8312cefdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/6bac3c8c6d6181776c62baccbbcebffa357487a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/aa109d11a59a9112a86f65e4eeea2b789b8593ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/e172992ae005f6f67bf03811512ced0f9e73a7fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190183.80/warc/5a06afba6d4d6a91fe267f10708a68907df19428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190183.80/warc/5d03726794439c466fb498bcd38eb8fa2b3c961d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190183.80/warc/75339a5835ade8f8ebcc9f8a7ffe3ca4dd672a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190183.80/warc/aed4e104721fe72bff1f343032882896a6a0040d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190183.80/warc/f0666819ed196533dbe39655047b94d192ec2d23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/03cba226155aad407101190a79c3d61023c5581e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/0411b6f4ae1d89edc1e6c4d401550aa4ace668aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/40dae6b51586c3ef2ce465cd73b868e12b647d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/62bd77783cab146047ea55db9f7ad4d788084f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/687d7ccdd083aba15165878fd5d28a3ff8367550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/be6daba86aa0d341c4f09979db4eeeb6514270c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/288f15f3542e6386c8896ab3f4527999a7744aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/65bf1ff7a3558ce3f56b8bc8243990b3dbf0cdc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/88fd04ce65a411f46d354cd7bc852ca8be819046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/a5853bb041cf8dc3f3fa31c5224acb8cfad66c4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/b35f259d449e862246f20ab34a6ddfed94928342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/e556bd05418d44bbfc77d0413421c1ffa2ea0eb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/02cf03416e6ef7ab3c0942a2cdc5a5bdf72ed9de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/204da00017f3aca54cefcebc4b5c0d076f112a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/757b24901f02e78b95a3c7369dee14badb044c8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/d931dbf71b6cb2e098e40698d25d5459105de990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/e116b3520117782a55e876655da71778998ce866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/ffc2cbf4a7e0ecaee51c40e084d6a3f402c87c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/4d2cb21613a4488c8d5af0e57045cdae35ad0a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/7557e30121f8af8db6adc71ddc15d6997edc2634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/9929c71c398bac3f168fbb07cb9fed18299dc5ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/9c05646ca885ff3a4e2c7c0f7e75c2339c4aaadd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/a44139dad92f0d43c51601c89c1a19e0b81f50fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/c11f7ea327907a2e3a3d50912f80ddb4df41942c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190753.92/warc/0f6437ef0a7beab866ad58fde2283f4138ad7f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190753.92/warc/5bf86ead954f6457aad0a4e4697041f26121688f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190753.92/warc/cc92c230b89c6aa4a07afa1d0d82b48deaea9924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190753.92/warc/fdf90e31dde4f3f460d6733f0a8aeb293f960c71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190753.92/warc/fea6ef8eba5b329520c0c168df18a183d07b3715.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/050e0cae1676aa0478da3c57a3761aeb24710212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/1560cb02552ebe34f6c427e52e389203b958259e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/401231bf63e109dc14907aa120c71d5aa4b9c598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/4c639a499649fd43f4afdaea8d374c6062f10479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/6400d9d8ae07d4e58100aea4d592fa2dbc9ebab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/cc8bd9e847def2575e9614202974961fd2b03d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/498427bd872939817c04265758e626ae1451523d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/c3fbdef3f40991e1e63a92dcd50fccf5a00d6635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/c602e504d78059b02a8e8a5ceb0a3b2f948783f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/e29a34e25f75050c5a1a8db5c81ac3e5203d75a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/e59a1a67e9a22aa53bc202c1ae15d2347108e556.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/f3a1d2902343cf65af86277cef208d40efdbbfa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191396.90/warc/18df5f14a3e385c003a8e29b2c08ed9f1cca952e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191396.90/warc/2436a3d69e8750ad524276d530242c71cf4837de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191396.90/warc/407547dad1dd838c8fc648153337f38c5626e2e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191396.90/warc/5739ee782b86592ae7d9ad5cc18e3ca767df5b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191396.90/warc/f2e08d5d18761d3c5a6d0892d8456195aa104d6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/0035468090db6ba8b54fb350a78c2be1fe9ba098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/2554c54a878bb38a00e042ce9414454c07c6b21d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/2a9b7d8fa01e156b51cd24bf45e5cf93696bcdbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/31a5366d9d17f08d9b1e77ae8b75946c6e73eeae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/3ee3927f0179341e6c73b8f0b7bb10968fedb32c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/5078621196798ae806de9c751e33164f2d5f8343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/31e72dcfa07f25eb5da9ae7e0165d2ba737e298a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/3b9fbadd0c121c36be48c5fd08384727d6916268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/4267d9117ef4b9a2d2306ab154db8b097bda400d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/45946d7b67cf538d1f764ac0c17c15c07101165d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/4d65be331f4f7d72fec76ea2f6ac0059b826272f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/90e673c438590e6198d37f4c10121fbb76e0555e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191984.96/warc/10d89ea8607fcdaa33f1b393e8c0aac0b5228a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191984.96/warc/7bf42418ed70704df3ee2f972f70d4a7f4809b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191984.96/warc/96ce049b62720e3e252097c6078c0c3109b2016c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191984.96/warc/97e73e2e03e2fd1319616889770e37e4ecb29cff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191984.96/warc/ae9cc64d622119df1615c919d1453c9fd682f517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/17426ee86c62bcbffe2093a2a12f8c9197aca142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/1a9925ceb9ea5f1c8b095a4b011d921d0913909a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/a24f4401d1477c47e6fc4e2e3d02b4dd67bd4555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/ae81a123a7aa691c9f8f9e2768ed456a4cee7f13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/c3d0a90976d29e439298e0444f43e396f5bc5db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/c92794ae4074a061e8659a843913e1333c76752f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/3ed2da9f9ff98157f49acb782013da8205893ec4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/4787eab1fb052da0dfe53c31530bcfeaf41e0e91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/5dfd6630a92bdf41f0b32bbb9b185fe3d810018e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/75c17e838675dbd4e967397ff1a102ac4bd0b15b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/7a8a9973bf03a62768b1be2f956c35a91f4185d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/c0b78a125d038f2b96931081f6db1067f97d1b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193288.61/warc/1a4af655ea263ba99fac856dc9949ac4f5a0b3f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193288.61/warc/26e562723578708dcdc4e6f533f6bd32ed297967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193288.61/warc/6bc5c04f27f43f39bf56790101029d1b7c2ddcb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193288.61/warc/d3af9c23ee6bcd5f34e8b13b4d7b95abdb656044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193288.61/warc/d503843ad84aa024cf617f1e8e5be32925b0ab0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/2b1c434382e5da365de48f30a69284523154517c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/3df93b3ce2cacb66ba756e7cc6046b65e4bf7328.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/78749689e03ac6353b5a0604cd446f7d57cd0a5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/8224c0bc2ca122418cf053d8970f5c4de1b09a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/9d823f2fef70fd09dc3d144850aad712d074e190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/f597f9602f47bcaa274cd6bb4352e04bbb49ffaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/0e732b3062dc4f0706cd2199c152e3ae2ab11ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/412e9f0ff71bc5ed950b315ee2cfb87cbb7463a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/6d452174220d044f498fa8d65263988a79fe2281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/9d43d80ac6a0d54484c17ce3fd0e2ac47d9ad1f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/cd45dc847b50414b9be34df8421383915399f9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/fd6be345fb921b24193ecf7a0d317dd2acd24381.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194601.22/warc/1ecdd8021445915d6710a148418cee870dd5e987.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194601.22/warc/8eba79e3651790e0b8941e00764caa308667d465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194601.22/warc/a754ca0953d9d11969f5d2fa593087f5b5814ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194601.22/warc/c45bc84b0c576099433d05b2c9aef5c8348c5e43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194601.22/warc/ffd245d7d671866be7668c3bf95ae474c30f72e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/23da4f590c83336f4a9a5d130b456a0a367e325b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/55159107402c70d3de532d4bded7a9e7d7afdd90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/5e0568114ccbbfd344aa404c4356a2614fdcbef5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/855d1715e6f06ed6fdee50be86cd39ef9be141f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/a69d62d4b8334596269b445e3d9f8cfb05e012d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/d94a0eb3a5396f844a22fa765c3083bf2af5a38e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/43cdd30fb1193a2af550a72709985f2c38069bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/6eb01176f43ce00e0731e297e93c038f1b853476.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/7757397d2f1788ae1e0455f0d72397dcabb1093c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/78e19c7eeaaebf78b725b5c03206268d6cd5dc49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/f130995d5a4f2b73de69de3b0d989776d888c2b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/fe576932007d06494e38882ff9d2ad9ca0b9b12d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203515.32/warc/1dd81367687d2e64286376470d318ae2895e89b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203515.32/warc/6b4380bf05d109c7d8d5e660122dea682e5c01c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203515.32/warc/bdd1a64915bcdb833c64ff4de6cdace8cc147c7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203515.32/warc/c43aef0994a0f9d98d2d1742e5de682b6ca108a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203515.32/warc/ff61db11e2969cd05bcbaabc68fb939fc7d59faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/37ca2c1266af2d8527877c191bb6bbec2fc5a8aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/3f7e5e91246000f54a907fa0aaf46e39ffde415e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/4b8f1127b12d4168ea6ef52ccd28749e25f83221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/ac7bd9176091c64f62028454d39ddec8912efbbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/c118b2f9e2fd137e2abc29112c3c0adf278b090b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/e86754926b94c8df2356d9f8c4882688b790a6e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/7ff92832f4c675c653750f224395b23892243025.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/8145e534ec86cbbdf033c163cab55ee98dc30a80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/8aff637b3b7b5e3ff5ff6c127a0f78c9704afb0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/b107113aba0919ee39f16f13f42edd02949e73da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/caa83b69d28cd50de8e8cbd3cad2c510ee35bafb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/def8302f2bf9dce59ded2c2103554120285623ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1b48d642e84311c9ae996c942e7248ba59ff81d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1c49b321d9177ad574160e1d9a998ac8b772baee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/211f8cbbf4b5ab3a9183099da15f56ab21d55954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/27deef0992f9cb9364bf7ff3c6e4912cce1dfb9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/308a58683e429b73134653172ea8c7ff74c5087f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/328ba47eb3226f38cdd6ff23ee109cfc027af7be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/363eccad88fa369d6f47a2b246ac9ac0bb7d95fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/39cbc7785eb5e2cdcbabbf5991c99d6cf7f9eaa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/3abc16aaf0bf0229ce8794e01046f4a8bc3c18a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/3bc74c72a20ee70ca47cf50c64b76433564f880a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/41003796046195857c472147370fc6852cae5dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/417ae479976d15dac5299348ecb0748ac97dc5f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/473e22284cbcc3c48760c90d6ac1566f236998a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/49c164fa570079ae0e33d68defdcbc2575416c9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/4a6ecf4862adb9dfe8d32528d6b78757cb7c8354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/4f4025060aee2ecbc1bf4fbb2e43186d15c04dd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/505c88c3300e38c5e586c2210397c94d352f46b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/5183dfe7d69ddd742d4f91259d6aa4e4ab510f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/52b65e4d8992f5defa6c2c5610d2c4a5bfff85a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/53f3b6177aa24b2c7c38054c9583ef5d62381bf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/57b9adb087f624c42b3d6d38463c5d7bee8c1fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/5baaefa9ece94842c23853f2bd35148f2eee237e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/5c127fb51b1fca5e6e6129b2139a230a52871a52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/61762a2d3985c3a19ecbf34a28bc2becb1c0e1bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/6176ad8e6d15d0f3adff155e7e1a40b15eb34d9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/62c65ee5dffe981d827a607b0e616bff680c4672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/64852b8e7fa0926768a2c2623bf5ae738787cb78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/65007eedb1d9bfa6a3f76ab840635ed70a7bf1b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/697925ac0dcdf10968a4a7546e361c74e7b194ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/69e7f33cc07710dba8718319599a5c5c9bb12946.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/6b420e8943abcfac6697470eaef1e842f20acd07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/6dabe1ddbb4d5275465cc647d63247baa560acee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/6fbe76e47f2fd8604726e1ec9f82824f1acfb67c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/73cf842e53293a0043987be55abd089c391fc6ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/74c0861d94ada315adf238c60b5c52a16a893644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/78cfcf95662e251093a1b31e0be32f9429fd3605.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/79ba368c189e3202a7c502e066f68371a4913651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/7a89afce4253b9789f5db1ed98706675ba9628eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/7b63f130b95a2b27114361e429242e94cd385253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/7bc01d3d513deb53b734ad56c1e0e56dce8ca614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/864f15a98e1261f12394fa05fbbea706472800d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/885a645c3bdbb22d3982ba1796bc78b18c62139e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/8c5d0458dd86314190f06d1de242380570b9abaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/8d98a0d16c79b546ca9a4dc70d2c10845bc71306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/96b58066e0d96b28774a9a95e0b33659325e4fe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/984018e4e54564e552d74122e133a55c2986eec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9a56946efec67208a44e504cb8a3f48c446e6cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9bd08551ba54c18ed4ccfe59091f3696676b9e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9c8d312d2f3cbe4cfe897f6e0556d7aa842d6ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9e6c696c635b3eeba4fa3b86f969f7d1580ac88f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9ed3f7f391fa9fa4505d9152677a34b9b909acac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9f20fce630c753a5da46c5f1296079f6a9e990fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9f5dd7fcd86ef95ee6ef2c23934fda14692fc36a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9fa9b0698371c03301b4743bd8cad99362c337cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/a1857afcbd13f39ddca729e6c3bb08dd03613604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/a1d303bcb225f5d68dbf96c0f54b994a97452455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/a405fb311f33373b48a60a159bfd4529b18d0afe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/a60eb1d04202c64b2c7546add3c1ae3e318259db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/a8ed0bf9ba0abd6b818346ce431df92ecf56ea44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/aba48f03e24bc22eee00d7e34a7f83f10ec27255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/ad33fa9412541ea52d76041bce30d60fe6261472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/aff4f83485e8ae5da853708965248f34ba2acb06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/b1fecddbb34a41c35af2f970ec02af5377a66626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/b8ac1c12805441cdd5d8157e17e998e003819e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/bba27e4ccdfa89c8e9fc0d8d40849ee3a23e556f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/bfc8dc885759d6a1b7228288d3f384f768b4a64f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/c197337a8b0a3db5494690f000574dc91e1a1781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/c636d395e4db12dd4b11e27cbc0ead48f98314a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/c78eac998b259ac6c649a99743b231ccdbdf6cca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/cb6dd116245b01fcf01e9a90628037a254bf2eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/d824a1d4b71d1e4e2310ebef782107b81a93ce7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/d8c6f1191b8d0a1db425c92706f01d4018ca2b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/dbb392da339f1fdfadcd8b233c3ca84be7b79b09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/dda8b36417a98326c81285a534acdc66526ce1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/dea441ca36fca85f9a5950fe5f78ca0648c3755a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/dfed0840a332b268da02f10633c877207d66d65d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/e21b64493b4a47f2d99f29e2ecb9fa702e7a1fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/e75592773d95ce93d4fc95ad8f7bebeb2de2cca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/e92d620825b4607dd33ab9f3cdfdbe9b104f0fe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/e9411906553dc093813ffe06f5059683c23676b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/eb54b266e894e7f4c2141bea0efd84b72a6181a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/eba5ec531844827bc94b3b4443f96ce20e4cd3b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/ec40b945765804b828f8003145cceb5a97275346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/ee5a647dd9c881a30bc4eb5f39274fd585b0ec3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/ee9cd411306de13dabd7573293b2142d330303ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/eea4e378f7424811272d3b49a10e3212eddc0b00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/ef9aa8c95a6dba390b41abe80349ccaf5d4cd988.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/f0ae83c35d8abeab7c0737683c37e9acacd75cda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/f321f18fa2b917084c7e5f322944cca6ec699427.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/fa37d42cc016d4600b2639ebb15cb35e1fefca67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/fff44dfb9d9410553c836d5f313c2f50ea4cc203.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2017-13/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2017-13/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2017-13/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2017-17.yaml b/configs/cccc/dedupe-url/CC-MAIN-2017-17.yaml new file mode 100644 index 00000000..37399433 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2017-17.yaml @@ -0,0 +1,668 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/02c7c648317611d4f5597bed741fd9cff8ef6d7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/05bc1e23cba40fca7f32abbc7d828fd64597c21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/064775f91f18dfecd8352ce6692d0ee7f6479855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/09dcc4f1e5ebacbdabd7029f90a67be38fc21ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/0c2912aadab4791453ead24e59eef0734151bf21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/0fe5b90428527d1b19b7a417efe2a927b5f88984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/10a6941eb9f5316ff91065e3a7c0efa6261036c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/12fe88cbc804015864dd2b5f41b03bfba4fd5a2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/132d7ac30acdde3b6f8e94e8085d59c602950cb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/1e30974cc270bcb4b92bb2aa8e3a0c02f3c7d687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/780fad3650fd65380d737b52816814f30102f33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/936f7e395ec243f41653a60439d42305df91d2b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/9af60dc5b75f4d9b321e2bb9d016e02df5853788.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/a16c105d0f89155dafde95ca50aa630d74af79d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/e2839c4f744bdd4b8f383937350310935f9acbbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118477.15/warc/3f3cbd455ef4a78ea2977dc4e781f3d3c567ec52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118477.15/warc/862dfbdeffde55a1df8d919f1deadd14ef74dbd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118477.15/warc/a5daeb1d9b35dd41aab6e799850e6bc47e5e7e1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118477.15/warc/d4659791e909b47d7af03b22538306959c054445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118477.15/warc/e8f1e098891ad6f7a895d18746a8b94812c299d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/6191ff483b6988bf209dc02b465e3cf6b102041f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/8f1d5743d79df8d2ffe481393a7bf4e427f8a4a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/a27f38db07268d4a53b3cc81be1521d72825b06e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/ae2fcc89bdb987cb8197869deba48e0066687c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/b80b6e3ed7d8fd7507ea95895411774806aa8feb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/d64d7c333b7e72bedaa8a5491f604104c2fa1617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118552.28/warc/1b4dfa3a88301a0545a431c878d8da7991be85d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118552.28/warc/1b71a02b1adbb8562f3fa3f3286de9bc61620bfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118552.28/warc/5282932409368126f40a5ad32a816a0dbc8a3dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118552.28/warc/8ee4e71735002947cdb61a4411074bb19d250d9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118552.28/warc/bd8d5c8515aa589485e513511d854031a0faab9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/3a159571ba6f233662f91ae60d6b1a7866078c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/400a6da15d86df0f7a23fcfe37af758eaff8b020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/b0a3bfc1b6d883ca84e87e3e743909e7564424f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/c22a328d0ac06c7b1fbb9f000f40a7f180093bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/e9a82827480fe6555f38c50641993cee47276f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/f5b2c6a488971b282c0e488deee87f12e7d9b134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118713.1/warc/21eda8b41051b1d256312f3506a685e6c25a69ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118713.1/warc/367b5196622416ac28b8c0d5735cca3118893831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118713.1/warc/6afc618dc3dcf0ec4ffca64aaa0dcc99b405e7cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118713.1/warc/855421ba62eb0409e12eecc92d3e1827b86e6385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118713.1/warc/ff9a688b52052070661804d496c35926d349cdc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/31b80d0b0df54a2a2c0c5f3886cf47163a0c54b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/44d679ee8f295ea173c6754862ff072a0bb11c3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/9553988ba93d2b8d7e8fd8d22b7985eb26d0387b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/965e12d8af593f189d8248d8fd105bb093407fb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/adb953c3bfa37df18aa12d295840410246c96d75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/f94fe8c8966da838d4cc8755b94b8e6f2f764895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118743.41/warc/139b42c2d154a3da8b3a540dace7fd22570a62bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118743.41/warc/9f4af231595cdee54561e72cbbae05bfec759eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118743.41/warc/b96a82792c0f2049f49055938a35ef22574ec6a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118743.41/warc/d6d5085f82792746f459eff6d0f8fd940b516390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118743.41/warc/e3585f202a7e82187a257b44b34fe303865e083f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/3f43d0a0fc67099023cc5d263fbfd16d09c01913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/a9abd0bb82df53660b2cbcae23a807e8cba4dbca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/b416ed64f5fdbe21c165ddfa22b0a834bde35deb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/d8003e06632714b70257f12a701b51831fa61d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/dd43297426b681d349de989e30c0584b85ae9b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/fccfaa3e588f216f694d23914cb3657575cf52b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118851.8/warc/298935d030feabc95af8e52518c9a1d5d5609d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118851.8/warc/b2b20a25c249f6e470131d31b82be11368f40056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118851.8/warc/dd0dab4c6481e3bdf44fc6cccd15d3c668099a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118851.8/warc/de16ea3e15eca3c015690c5831cbe326b5ad26f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118851.8/warc/e605ffecfe9a425464d126920721d03d9aae94d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/45c0069477fab0c71c1a7c2d7fc198398bc92c3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/644eca93e3f886b3f8e93062e4ecdf9cda8038d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/73c4036065c8e4f3057df7aba86f5be2835a2f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/bb79966fb34db4e6addc556ae475b979099a4bd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/ce2665d462abb1d9881860a1926323e8c1fba078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/f6f453304753a9bca74b0dd1711e6649962fb8e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118963.4/warc/1c2f5536a2e0834ea72cd6f1ee1e8e4d79622bdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118963.4/warc/4a65af0bbb5544c7b3af86cdbc262c7516a41590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118963.4/warc/80bdbb3bbd982f83383f96033175d3e874f3d6b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118963.4/warc/b95b02880365971b8e63aeb0fd94eab9d27e0000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118963.4/warc/ee4147ef2b6cb06cbdcc9c3f763c6c11c90b21da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/26a118ed3cced4341d27a6752fc764f20f976f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/4392943714bf3dd08bca721314134b2b7876c339.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/709062d0126eff80c4395b78be125eb59abfd146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/a7c3324067de7b2b19fc23b0a63831a3c5b8ce19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/bbfe5a92ff9a6f5203094cd7e107916979e7ee40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/c99d8d5b9558c23a83a155158f37e822bf2ea8fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119120.22/warc/2d265fbb49e50948ce3dffa7a942abd9c8b90b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119120.22/warc/3675eb9ac6cd5f79adf5648de394191ba60d6186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119120.22/warc/5c2104f4b29852664b4851be77430d7eceacf8a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119120.22/warc/8ef6cadfa5c9f6698fd13d9d041bdc5c6378b64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119120.22/warc/ab185550b97db9f288c2bd3c823e71faa86f2125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/4120293f50e1eadadff5cbcf4f447fb1b403a182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/69423a56df200bf2679e897ab55e2e0d42918880.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/7c65fa0dbd71235b318708a964ff5132a2dd8d65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/b27b46d2a50e3b8478fff16e6545d47b74f6c605.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/cca2370b48d9c862a4e2374823a11e0b999f5993.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/f0b9f68b024a121702f09b3301024b11ecfe0660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119356.19/warc/53394fa6228f4b619d73010cf8042b25a68a1af0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119356.19/warc/58b22adcb8194de4f2684b8f5e1d90f6767f2dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119356.19/warc/9242d6378e00cb07cfdf8ebd29b62e555f8d9364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119356.19/warc/a72cdc863184eb7939712e81816102f379a0e703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119356.19/warc/e9d9a94858b9cf8b15a8a97388f201ffbe18fbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119361.6/warc/30a693e997b03ce75436f4fd198d0f92fd669769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119361.6/warc/8f5e55eda6361f6a524da728c1ba853b1358705a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119361.6/warc/8f9cf0a14d4a66647563c89c71b118cd75eb9899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119361.6/warc/a0a4dc9c4441ead6c7251c19db58cc29ee7b4e91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119361.6/warc/fa9355a942350960c23add384c852c6df9a04c82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/212d3378a4bf0975ffee343361233b9f84d87dfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/409d5de7b2530af726b4428e951059fdbe079d66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/66a49652e51d667f0f7889db2a30b41c3c11d57a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/c7c67d1e0c2e0d49ad5f43d2fde4e42ad578d574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/e0e461c7a3bc2a28b906ab3e65623d86ab987e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/fbe7ec8219aefc23f6d272fedc58c287088cc345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119642.3/warc/1a3236a41901c80ccdba590d735ba84546073610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119642.3/warc/334017ced465299ea2560516c272cd54cdd30105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119642.3/warc/44680bc56971f862f21cbe246995f32811c3cbd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119642.3/warc/4d182c1907bd155c68fe7884a62b04d741a75e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119642.3/warc/a3d0a89816ecd9667f2eeeae2d6b45bf87567e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/0e3d869fe1a7e06486857f1a33613852329d5791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/299aca65ab386e68c721045926b7dd00cedb72cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/52465d3b41bc1f836bb30ea3edcc7eb8ddfe9146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/909ef8c9e3313a1820dbfd2f8da809747e0dec79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/9369c0db3761a275901103bb7b76481f734c3b7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/9995c06e18b225fd3d8bdba08de57dd7464f868f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119838.12/warc/1ebab772957937b97415182b47be715a14655afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119838.12/warc/2d66a43119c65dcc651260e6dda0679426d9936a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119838.12/warc/4a58cb56726f41c210d58b6c3a5a413d6bda504f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119838.12/warc/6ea0ba60b06ed3d07ba700cce1d523b4fafb96a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119838.12/warc/85d7600f8642180240fc1c17838537f2bb0df219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/2d57009a3b66798b7a030d818309ff7f5e284e03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/3eee26784abf8fafc750024c739377e8f3afe09e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/58300d2049b9f5db0ccab41be7fbb0680eab2bb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/92bae70cbd595ec75dca41cf4e93b3254aab8fa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/cd33004f84e1d9413fe05cfdc2e6dc3e96a92fe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/fb8daf54689a8fd269abecad22587dfbee362b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120001.0/warc/0d8b20c451e9f34c0193a512a7b841a6087178da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120001.0/warc/3db3816b56f295841022f094f055c671f0c40098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120001.0/warc/5b065042ebefd54ca0920d58836214a636394d0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120001.0/warc/8b73db58ba30206ff9ecf49780ec98adaee46c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120001.0/warc/8edc0031ebea444242ebb21c813a3d7020205eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/167dccd11f58d516970f2ab6b1305b719dbcfd73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/1bda84ac88945aa5959555a5d1c7cd6f65a795e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/22809db6cc69cb3b5d9e075527430134ec9a5d23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/cac34f7a818c13e05aa9b7a649531cda8b38ed05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/cad3a39c4841d252df5ff494cee1a00684799b84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/ecec6b8b5aa0a3629ba68400c073d95d3e863926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120101.11/warc/1c33418910e9688a0818ed103691205056ed18e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120101.11/warc/383d0b53fcbbaeed12f4ccc9c620e7c302d11818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120101.11/warc/967a214c1efaf5b6ceb125853c5b8941bd09616d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120101.11/warc/aee5bbebb8284a7a78b6993e01b3c4fa12cd2c1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120101.11/warc/fd4653cfd784048dab7fda98ec3324c8cf38af22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/11ec66b01a926697c63d1fd06f5adc072a4df000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/2ec29d2a2395c87dbc32f8d51c28cf62663bcc51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/ab2e03dc363a8ad6ab7a092ddf8c4ffc24450262.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/acdd13b896e7011644f7f9956b25461b384cef7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/d462b4e05e19b031c7cb46ca7c54f28dbed6e153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/f81c5879aa03e3587b225b04a93d3f945e1e9375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120206.98/warc/4c51a7aaeccef55eccb8b2c955c08a34e733f1a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120206.98/warc/5a4e25864b37f1044a490621cca4e9d6ed8d9f11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120206.98/warc/5b43363cd2d1e7f710aa262acd935903a6d2fa0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120206.98/warc/6f96e195d707df2e760faf0ce5a8dd8ce798faf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120206.98/warc/e7d6ae6966db54fd8a462439f70b3aee26088e46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/08e429c689f10a3d2bbe03c04a74fd222a496633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/131af115a039877ded95a4d6584f2022ab5b377e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/2bb10280290e5ec58f10c3e0467082e217c20156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/2c23cd1613d197a489fa6026eaf519671a8cc737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/51a164d4c2ad9bf0cdf3203b8e60000ea72ded0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/b014a56b82b78d667987989bd939339d4a750356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120349.46/warc/3d202f85e895bd8d2f78b272c5932ec9d5a5ca76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120349.46/warc/8599eb2256931850b68d53c5a3ec84100ddc6c67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120349.46/warc/a4aad9ff131e2c189b48555757156cccfad326dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120349.46/warc/b118c4500e6cab054be4657e81e3a1de29948edf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120349.46/warc/f63b304c0eea2e826eddd1ac97193230691319ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/1e3a5bc6188164e86f35b296efade2b0a420a792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/520fc6398d387363f658a7acdde5311e64a17a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/5c60adfe066be25c99c3b30ae53160dc33fe9db5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/6a6efd2dad043d146636bd1bd1b069314fd7ac10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/917d1d1373c3e8c56528032efbdfa1b56d9a510d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/daf5fd5e1f3e32236d309f538ec761b9298b4714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120694.49/warc/7bed3e5b45ff02535ba2bb4875e78faed34042ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120694.49/warc/90724e441d52db789bc7c92c6e6bff4bc2b30b69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120694.49/warc/95d9b708e45c53961edf880af9d0d4995612d821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120694.49/warc/b2c8377fd3ae68b5bddcad23562d9213ea3f839d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120694.49/warc/bd0b192045d71b8066b55edc8997bb05ea303836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/19c5c15cb40dd09db020686ad7a7e8da7bdb525d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/9931fb9d9c64bd14141e15ab21295c4130b6ea44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/a9fe6e447ba8b7462d7fc867124b5fa4e4d9ac9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/ba7bf4a107a8e59bb920b28cc5b5f53bf5160939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/df78de382363aa3c5629b8952e667ac1bd825703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/fdfe0303ed0ddbd9481f37884df5319afc5ef286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120878.96/warc/15b2a9cee377d1645fe4a600a09664eb434a71aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120878.96/warc/328a46f6478c2a9345f8a74c6106821ca7022273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120878.96/warc/8fc77b64f10502457c543e7e4b8d43430fab12e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120878.96/warc/a84e388ff4f5984fc3b554c27b11caf867c67729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120878.96/warc/adb10073fb338693b26843e8e2e54e72e4ff3ce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120881.99/warc/275d646868426a74bb02f2abce3972165083e1a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120881.99/warc/33362a8d145ac88a322c5c48d9d39bb4051b9fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120881.99/warc/636715f02c738a652d9e0eb74c051313a52a7710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120881.99/warc/7ef50f1273355ecdfa387da302698c2046e59d59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120881.99/warc/b0dfed67dc116b10003d92071891640f410df815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/00b67c10bac743c2baf163242c8de6761081ab53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/0b38d4a5c727e65b4644c436783c3fc49b145257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/8a18fabc5042b3695cd0d893f8f47c4dd054984a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/ae988f2679b668c0bf1370ae63e5e8b2f42659cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/b959db1515ff77f0cb98bb04e2d22e92fe9c25f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/ef04fbd091e295ffd69e051da711fac91685deb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121121.5/warc/78a8417a1a190792bc709d13cface226d497fbfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121121.5/warc/798295374a8502dc4b5150cfd691195f30353ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121121.5/warc/88b668f5d6442fb3bd8e98008e1d2bdc02039068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121121.5/warc/bd131035c70543cbe684982f5744b720d9fc06e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121121.5/warc/cc0b5fb8f092b60f49dd0c6372f346ef67c2e038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/09d762c75454235f61bc9fa467d32ed7e3ef059b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/5583718619bc0b04316100cf4d41ced1c48ea36a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/b14f55f5f4010d77f463279a1e6da0a2ba1a4f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/c0790b695a3b9603e2ed2345582fb024dd7ca881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/ebd68d957b1231ac60fb6c39225d8aeaf484d094.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/f0a07381d1bc54324c112ec4ca477a802d621302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121165.73/warc/440eedc06c00867424feee6a53938e71db191fa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121165.73/warc/9db274a3d42bce6af92f7b3a9ef544555c135ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121165.73/warc/bad2f4558fbd28c195cec921b95c4cd42b2f90e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121165.73/warc/cbf994c659850ca2a8c4381b893e374150ac5ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121165.73/warc/e966133aeb0b62acff1ca9219f71445978627e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/2173e5241aa7479665414d826a4c702c8dce0d89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/2a7cfec269568975d6725031034b511074b54e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/6137348cd18158f1de3fb426e3f179d90a5204ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/70aafb780bd5674f27538ccedeb9671d770c0405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/712d7ec26d1258cbf1b5a972f31cd2cba6e863e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/c20018fd52090356347eff5ddaef0f819892ba4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121267.21/warc/3b871e3c1d69e74254fb0106302761e1372f18c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121267.21/warc/80c3059368f2874bed7c066d9aae8d3f811b5156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121267.21/warc/8fb152b36999084b2a9500677dafa037413a00a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121267.21/warc/9d3063eba7517a24990d29cc1504ca8c06f7be22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121267.21/warc/b5779c8da0ff28f4a0240a94f778c4b683261aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/9dda1646fd87cccfdd4cce5f7d9bb289519b71e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/a7929e54844f9ee7a4e7aac2a66353b469c54941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/bb6793a7fbcab2673cc8b0698136cd41741f2e8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/bcf7ac75f999b92bc38988c3da74e92da087e2bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/bd2850bd6e709998116a6724f4952f52eaa19989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/d477b01c0f1bb96c0a13fa43a20645e0add80513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121355.9/warc/0b66072849c82ed30e5814f2d24547d379d47730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121355.9/warc/2d6c7284bda557461f61b236c0d0c4262d978b10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121355.9/warc/7e61e5e790dbe18551db610dff2f438d6feda5c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121355.9/warc/ccad15c550985825c63d96519e2351634e4703b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121355.9/warc/f6251e0119a4a431ea6f55a455ce36a54287d21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/00ff170809393ba11b23cb86a9c34b3b45fc0c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/0648ced50dbaab4f79bc2b56e93805ee9677c1c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/0e7995be3141be5a6755c8a49e5d83286b1dce17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/5d0560885f4ccf63ac7f518cbbf8766964e6a7ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/676d93f2c6e3646135c0f68db18eb269b1a15670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/8ac2d84743d734236a7dabafbe60b30057e84ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121528.59/warc/4223ab9371b514533380f2a256c731ed6d309fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121528.59/warc/5360d31accd7505d44103bde9f1c664e38e949d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121528.59/warc/b32b6fefc9feb5e02983a09a874c1e3c8d09750f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121528.59/warc/ee3c059ba5071570e6a3b2aeab3f51c8a0c12c00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121528.59/warc/fcdea448d7db217b22a0016ed6c1c5f45ab6c45d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/02ea7c86a2c8c149aa1503fa0215490fa8ab11df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/571c9e0f209d53b0eb7b80896459e26c6e0e4939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/805f5bfd8e0c78d31b9f98884b901477042986d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/bfe31158c36d3777c1cec56d16b4ef223867f7b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/c2a2f388ea536630e434e43a68bc247ea6c00137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/de946c624d5b20959365a24c82c3ced6cf0e2d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121665.69/warc/26725008b7f5188dd44163a2a2e9e3018e792f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121665.69/warc/3ce94fad019e7d79f2a56cfc3429dec605c94623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121665.69/warc/4cff4c80c1d355f2f0b091638b7980760d1fd9ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121665.69/warc/5f1ee8ae67240147216f0737cafea0d5446294af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121665.69/warc/7381d46543ce0230b3ba37a391a5e58a0246d431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/7db8e3d496b0c3dae6a435f56889e835c66578c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/99412e4ac9e7d46ddd6c531f783c1f58f03a1aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/ad458361aea5aa25f5786af81de042a30282c720.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/b80067f8c9c07eb3f8a754b2527afeb5ae4d58fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/d5b0a7ba3152d7b6754324f15f2a0cb5d474f357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/ef121cda35c3da507da360a72d0a9eaa01f09669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121778.66/warc/58e4b55449797ec20d7c49050ad3ae3198f88d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121778.66/warc/63258926c9fefbee6a6c446a0609569b0c86b40d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121778.66/warc/8d96b2ac401d9ce03788e9fb084ea0a41baea7e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121778.66/warc/ad0d9f8ec0874f5d8297003fb27a09d039ae534b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121778.66/warc/f9c3d1fcfa2fd1aac08b12f834119b509bcfb026.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/6079372f83f0a29d564d29297307174160d82995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/6961d496df861562b2c410c6fd6497e3fea3954b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/860eb4538c75be0774269ecc2d617b8fb44cd63b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/af3ab9539f9adfd424d928f89b65f7df3333001f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/cf6e36ba14f57ee856e067b9881042891b28a5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/fa1488f9bbbc4bc6dd119bf2bcf090a955967aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121869.65/warc/31843d40239abaca0c2e1f932b8d03b5011e275e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121869.65/warc/743cfa1b34844f88fab9a687c09c37b94ceb2fd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121869.65/warc/9028e540086ce8277f1c12014d5f0b2c7624e9ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121869.65/warc/b2137237582ccee1e6adbb570e8238c2c1947826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121869.65/warc/c6d82d0c26cddbb357e647d0b9886a8f2af4a981.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121893.62/warc/4abe886e1ae71f2134690eeedd356f5790022157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121893.62/warc/6f6465337450a84d4f995cd1b85e3270171f663e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121893.62/warc/7c250b69acc0ba9fe4fd87fb51d8dac0aa32de6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121893.62/warc/b4dfe3faf4163a3323b016d778bbe6e2e81a935f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121893.62/warc/fd91a6adf314a97572f89f00b34ac90f5bb09fc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/165c1ccf9b40ba8520f0220c847866c32e1f36b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/51552cbced513179d68ed2f724ca5ae4356faebd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/556d0f6d372e33743365d81be344140715709786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/56c4a91a34b3bbaf471f740843956cea8e73e757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/81f0bc2ae2c88c05fe42ad6b473f09636ea7c64c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/9289bae4f4b15b17a40e16633a3f51a03d4f1b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122159.33/warc/37dc26389414e39ec04dbcda9c378487075c874d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122159.33/warc/8980181e14f3879feb23e6e18f0fd2ae0c013abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122159.33/warc/bd16310e7ea7c34fd5eeb1981870d4651ce7da74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122159.33/warc/c221c732dde859fa19f75bd07e0ea44664d744bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122159.33/warc/c66489d570c8095b2e2a2de728af7ee809b6308b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/3239dace818c04952119ac697d0658a7a18b0226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/5556e0e6c7445dc411c4133eadedebcfd4dabe8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/8d6526b74eda858760f0952c8e12e607a2dcd011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/90c14b9249f30538aa8f652e00992d23ca8d77df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/e7a92249d1f6123d47ebf8a3636d097460a5edb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/f3f802d11ffaf22109ed10ce2c1318b18074bdbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122174.32/warc/579ee78bcab1ca0b7970ca3e486c1593c14757ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122174.32/warc/85d354b86fda27c34d6a86d46627684f7eaf9f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122174.32/warc/b151088710b74953ec182e8b05abba3a37a3bcc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122174.32/warc/c5032d7ba02f0422624e36bd38bb775a5b8aef89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122174.32/warc/d30e060317322e5fea297eaa560b4cda70c5cbcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/0eccab924b40173dca53aeb90baed70d0ab1cf0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/18258eb3c87e2db6b84329ea9f99150f33e2d783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/2c0a6c9d7ee0fbe100821919b6e60cc337f75c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/bf47764fc0d8e28b1d6e2a63d1909a00d616249c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/dbdd8b9ca5939064960282d6f89b56e9daba4505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/f7f64ce3c4d4236829e25df86fbaf6d91a745e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.71/warc/56f6be9750159b638f623b890f0a9cd9a2934623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.71/warc/9ce6110e06848a0b099eae818dc202b30b441eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.71/warc/b0ab40cc812e11362e93cdac5efca2be36412b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.71/warc/d1b46ae775ec0f7dc1ac102e36fa44f29cc69bda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.71/warc/d5e49a123f72e4d87cb6646572d686619a501b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/32484fb07c6a06d360ad95b85d4fb737bb2425de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/5e43698d2f3d9d86a4167cc2bebeed235541ffb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/7c8a6a3bc48b78e574d80d1f3f16be1bed191196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/aea04ac30989216280f6bbdce8a415ef31d75658.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/f46e209a559f4a000e01fb17114658aa978a01c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/fd562fc6354783116bf2f0d71e5de2ed69845454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122629.72/warc/0feb5fdfbe2eea150a1c072179d9af760cc012f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122629.72/warc/1950c6566239e31b83f64c05ef137701cd9253be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122629.72/warc/23e0d1ac9c776bb022f151e522e830b11d111d89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122629.72/warc/2a50a960055a2f3d5956e75493780a6600c1cc12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122629.72/warc/5126362fc34b6287930682fa1be04b4f3087cef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/1246650b7a88a94c7e16951a2f7a2aaca62d37b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/2d39b842b1b6cbe3880a66eca257eedd0f578b6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/364554d1a5c4f5955f6f607180889fc2c4e6a122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/4454da0ac115c7d4a719dae974b02fd251f4e688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/e8d14b63fa568585bbd56a2a9c204da4aa4979ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/eb6281dc1b522601d9e958a77c85442792b45978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122726.55/warc/822d4ccf3923067f248e951646870ea2b4770512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122726.55/warc/86b0233c095f5bcafbc3e2a04d2e9d75ec3fcfe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122726.55/warc/c25956391168952c94da5e6bb3fbc75002e92545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122726.55/warc/d038f1c902ec31921f81e5700a96ce37fe088ae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122726.55/warc/d8b223f3d8b4a42d88d61dd1400b08e4ff0ec384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/0121dbb0e6f1e3cae369fd46a18ae1756b3cb49c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/115737c79fc38dba784241ba8be5de3e1eec7410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/2c33a30582768cd467b5fccc791569e00c009411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/501186fa10c226ee5cf6a0f540e39a924ca2fe9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/5724211c5fb73e785d424a0b3b875bde2f141525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/982de690905c9e70b4a881e0e1f15a8cba09bbdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122865.36/warc/3ce1c148bbcf7a2e41ec7abf9fd49c53527c8658.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122865.36/warc/8b5e8a0302e132de37419600c46f9731dab20063.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122865.36/warc/abe76cbfc1b54dadc15d92a54bdf36a650e2a118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122865.36/warc/e69de5eb9de7f9b765d09e7f62ed9a205b16d63d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122865.36/warc/f769bd89169427713185e48b4546b1fac863023b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/23c38e97e26bb57da3ac5148fc36f0617ad74d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/462998f64f381b94194b54633511a06ba18e084b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/731b7f819b7256cad2af2cc58bc87499e70d775e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/833d7b7a6e94cbef0bf897eaea427cf4c98e13cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/badb3550578a594704ecbb592a4cf8bc65ca8ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/cc25d883a8693b83c371176d7cc6ab78fee51c90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122933.39/warc/37173d884a46e03ef5420c95988aa2ccc56f22e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122933.39/warc/43c03715b14b7419bac164ebc6f64e013820e7d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122933.39/warc/4d9bcb953d90a6ffc81e507e95c54c7fbbe97a9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122933.39/warc/793934979c937b56e9903d1add265bf748f9b86c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122933.39/warc/d2ec7db0c1b5db7947998e39f5b31d38fd630ce1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/428aa845bc951af0699b710152c3d227e95f1172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/57c4dc8faea348d9c4f53bdff3df3473d437461b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/90fa9cb3aa208ec0236ce0dabed6f89dd1bf6cd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/9be0202b241ccc04c16433cd07d9eda33eeac24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/a4f5b850999fd2b59e8e36b2b14c2c91a3dc325a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/dad588b552796a91d1835b615578f24cfb5f08c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122992.88/warc/a9680854f4f895cf2c27ad3e117fc0fb51217ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122992.88/warc/aee63e81030ef7b169c6ecbbaa1dd046e9259db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122992.88/warc/b70f80a827700dbf8735f69d4b9b800db25ce803.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122992.88/warc/d7cf5063147793be228bb87d3c6dbd34096c30f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122992.88/warc/df37e1905849148c7b362dd2e2dc65b07e9e4bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122996.52/warc/17a6a1460d82ce155a919f145486b45b368c5bc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122996.52/warc/4685a65ecc65ca64d3d79bfda0d13d29d86feff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122996.52/warc/795a2dc839836294310afae50e64684dbfa730ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122996.52/warc/8c56ffdce733ff3c3f1e8846757fb3f6979cf1ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122996.52/warc/a8a40b4c9c0a8388ed019be427e38a5340e11c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/219f9b231a6a3082bbffd88dc8452f3f382d9335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/2fe1ad1af2f6f49ed58ce317765668442876c83c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/57cab6ec81d12ddd0ee52b9171c0f4007c622cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/923700a4a919b094b09cb06a1dca9422d9a40890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/a2cf16c494e22f5e2e532b38d6dbcaac00d44c21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/a4dad3474c5b46bf9340afa29f5292b43cf82726.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123048.37/warc/215ec8e4fc358744eab735c418083829f7a25435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123048.37/warc/2f0f9c124b54583e12271dcdf74658feed6cbd91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123048.37/warc/5941334544478e5745d263112dca6e57b9b44b52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123048.37/warc/ba7970d0071504d0d5a901473e2e8a1a5aeb7d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123048.37/warc/c002114c388c68ec92111097b4ebbe7c4ea3bf7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/2921e8e51a47343b26c6d56dc7efd253c48099b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/608ba758bd1f35b74f8db5de617a1c749d67d8fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/6674644da93338e8d7e85cab93429093317d0e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/a5eb11975c56e34cb0a4df482bf1529aa22af0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/c2be0c8b823a564a9b78132dc16e428082272b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/f25a0c463192d8ac11f6ca2877a1841d179a36d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123102.83/warc/4f20b7a2bf171c4350c66ee03820581eeaad502f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123102.83/warc/53bfafe55f64bfa12213c9d7cd6f429c74bbbf4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123102.83/warc/5a8ccaf3768d50562e22edeefa892d8bf25d306b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123102.83/warc/68ec74c5e9e21a85c831682a294ace2580cdf566.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123102.83/warc/8b6245898d1e398e8225bfec52f7845d590498b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/45fe0dfedad9c1b5ea474bdd81e5acdd89bff19c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/4fceacbcf89ff6c34ffc0e1a4231d63d2a06ee70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/7a500435a7b1fb2c6eb1953fc5ad0121efc0c229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/aa5079cf1f8477f08ca6933a3804a4f0e30d152e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/e567369ce85870527c60d4ed7d0e973d5abc6158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/fcb95a72db8c4dd3830a282d074d12a04e57da4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123270.78/warc/16df943dbc9a61917c70f39b9ce0e88544582612.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123270.78/warc/88ea5db35a73f20531bf512bc60735c75c18acbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123270.78/warc/96d2ee1fa14a7415d19499bea19f813ba3e528d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123270.78/warc/b293d264c172984b1ec3efdd24bd4d22a3ed0efc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123270.78/warc/cc61ac2a4ed19600fcde447ee5391b61a7546e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/7da6266ff6e0ced802bd4063d5785aab850ced72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/844a6150025bb37e3f3db3f67f0140eb9d5cc0a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/cd2db068da3ef1788a70c40bc91771bc28b126a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/d11a99c4ed38b4e910a440164a9cf001118e1fff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/d5c99f13725872f39241966fec934f04a0136bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/f07757df11c0029642d85b6d2e945af25cb390ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123318.85/warc/1a1297065d7e632c923fc3d42d6b269744fb917e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123318.85/warc/2f7621bad8c46a4c6f50a8c9cc7a0d19c2e1e551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123318.85/warc/af35d0f57ac290bcbadbefb7f2a306ff860729cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123318.85/warc/b4bba600670832d61bea8969654da30e0a980cad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123318.85/warc/ba1df2beb37792d1958439854376348e05b34e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/2a655b7f90f8ec6f82133bb9eb23b90664382785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/59ce97aedd65fcc7c287cb88859c366b50ae6d2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/9c22cad2000ae9d61ff0330d67bb020b86e6ecc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/9ef7af2fff00356a648b989cd55900d462006b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/ca9d1f38b086f292de592b4cb73ab03d57cfbe75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/e0d1cda07fa31f906112306fce0dddb1d0b128cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.68/warc/13bcf53f8c0ae0a2b712fd5dbb20eb613ee6baaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.68/warc/21755a1c2d68a9d739fa89f8033e479d9862add8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.68/warc/5f8d771acb04987bf6877a89588c145580529c36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.68/warc/a210b47aa1dd490e4df5f169b6812e8a4734b115.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.68/warc/f789f4a463e13686e60b7220e9c4c4dbc6dbf7e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/123ec4a78b169adc476f3f0e6d46a1dad6f1c473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/25f4d6f733afb4580122df0827f8038e8b487f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/61560fe07d5ce786f41aa3949f1ac30e3510d897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/845a15c77c4392663ccb0b856c19cad84cbbad6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/c53d9420c8a6d6bbd4705981013f5ba8da90d5ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/d2156534cffd3e16ca8447ef7be8e7893dd858d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123530.18/warc/6fe3b0d8dea5e7b64386bac3626bca662d4cf56f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123530.18/warc/91870a76abe2517a9bdf5d168fa7e162fb8b82f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123530.18/warc/97ac5c7735718b229d0b8286082e43489f32d856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123530.18/warc/d6583c8ab6f2398778b4f0acf5b16e09df116b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123530.18/warc/eca92a5de40f9ff2379139018538bacbdf4c5985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/07225c86849f61cbdbc09c4562304245568fa56b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/0f63f60885f03577eb96a3af2cd88fc4d378253a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/4d09e89d17a07bd4c61b8a26b12e0b84b9532def.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/a5ddbab72663df9b244aca1080f5dc2a261f8ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/a7e109534b0b832eada0c478cb6aaa03a7860ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/b7405a5b5dac3d850130d4f8e519a6804828ef46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123560.51/warc/0382a1bebc387e4c09d57f98ba2622ce76cd1d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123560.51/warc/191585fabdaa1674a1f38f2dba3d22fb848d0911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123560.51/warc/314405a07ef917545975fbc286948ff7af6b45ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123560.51/warc/a61b16225757c721c7224f495d1a12c9fb3ec704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123560.51/warc/bab0a479d9f8b7c49a3ae5d838b21541f06dd777.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/1714ac9943edfc43156308d14a017f966d95d18d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/9f42c72ccec9ca8411cb80c4269b17769c3b5e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/a5836083e6d0eeb0583a7f7b4605a30cbe4d8c68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/b3e67fca0cb3c83862a14508d46f9418da4345a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/d5f77eeee345865000bc4f0318bc6028efdd9558.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/e9fbf8e4d26999e0418d966a0c129afad72363b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123632.58/warc/117dc3d610c0c6d88cbd13ad562012d5a2c5f4ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123632.58/warc/1199055806469d578b3e7013e10e5a465d286eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123632.58/warc/3f00607595e06b9c89bc6e78de7ba7423c8a5bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123632.58/warc/4d48a5c45208086bc761714e7a4e20440f54d509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123632.58/warc/b0ee81d692e7a9873442f4a545b64895eac191c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123635.74/warc/405b86d3fd79ea211e9559b1fd53bce824314722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123635.74/warc/73ac785786f16ca0d482564cd815d2b0db264fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123635.74/warc/86519d6fad958871816ec7cb7b135136463ca252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123635.74/warc/ba840906280d822c3538ec2e364474a34d9504aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123635.74/warc/ef049ff4d48a5f4c30341e91012d65d4bc16a64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/1413fe05600eaac528495c2d0046294447fd726a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/14255594053039dea2e185b6609d89a0c935944f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/2b899c6b89ad8298a378ac10d2fba136c53eb66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/4d4f8749ebc72694d2a685b7d8aecefaae3baeb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/74c0588f5769c281f8bf7ba07ef17b317cc2df17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/ac921dbe64deec40f67320e342bb7a75f4b7e916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124299.47/warc/40d23872f257a720ea7d0677ff9dc8b0a5134dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124299.47/warc/49f51b8d19337653811fef45113a1a12545a3387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124299.47/warc/549fb8eb60eda9ab53fdefc0dcd2935b50887ab2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124299.47/warc/990025c57e735e8fa57f74c2eab26698632b2357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124299.47/warc/fcd4578e5c79bd9f00161a42431d9cc688c4712f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/140de82d295bb305d5c9ad56c4d0c5475128d26c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/2c013882ae9a9b388a218fa2bb76ef62ddda59dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/4580ed9424d19393f7320d0007bb92099a8c6e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/9338956ed188c61cc96f4e7098eb93d048d38d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/afc752ec2ebab1f7b3ea1436f80e56098a60033f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/de4435cfcf8c9a739080aa3aa3b25dbf3625a3cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124478.77/warc/0b8e4460a6ad10398cdf783102b380f27f674aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124478.77/warc/6fc4d6fdbd66c67f7d5e5617646920ddce43cda0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124478.77/warc/777c77523ce19c7c0d121b6e9944debb7f6c92d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124478.77/warc/a05a460d8ba82d70b9b2119e796337dde13d155e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124478.77/warc/d940d393c1b17a693b7fcf03225ec9cc9b908884.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/0cda6cbc7ac2f5a38f3000ba2094e27c96f0a0e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/4bd2969a523c187352266645f799691037cc47f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/513c8a7553406a1f82bbeee5ef26e8ee2edf023f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/b15853a61e7b56b8624c8bf8919f73f481923618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/c385c83f4be81b28d9bd465e79fd9d2eee9aba9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/f455d3c42a4f7204d1c1af4c507110138050a4f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125532.90/warc/217eb9744f8e92635610eee95a0641af7c008df8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125532.90/warc/3791038fff09f881e23460616a7e6aba3d8f53c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125532.90/warc/6a0a7c8a6e455ca5f8aa917880c17fca6dfd055d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125532.90/warc/c1185607664e2cbe59ea88c8ab13fed3eb269feb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125532.90/warc/d84718ec38692e9c0c69b913824b8e541a19f8e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/0f86e87623a7fd1f1e1703de1f53d308240ccbdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/10b01004e1e8bc1523d1eff3cc653006d89dae2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/76c38b9147be727e79180a52e669e217dd257bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/7d3e7aa3b99e2d0dc7747a58fc8194e84c90f652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/ba6ea1982ae51f8abd0be46dad03e50258efdc56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/c4c2f98696228ba1eec903e1db5472931998db84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125719.13/warc/204197ac8651ae88bb7e9f4730728e88dc1fbbad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125719.13/warc/8617dd8ff41bc109210bf005df1d192a00c62631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125719.13/warc/a049f9a1da93655e5d303b80534f00db5df3a156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125719.13/warc/b59f5638cf3374d8fa1527f3625beedd2489bd6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125719.13/warc/f1e319998d7d1b40ee1b0a98751d33dd683294f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/0096fd557eaaaa3db5f119cd825e718efe5dcbc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/5d7d2d9c7fd6eca1405e46e53cdc601090a2dbd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/81337b158ac50fb76e8af2e56400269b1d6abf4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/84206e86f2ea2e53a8be26528b1c10fa5e06648d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/8f4768d1008136617a4dc8c3cbee0158a763e312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/e86277b3ca249ef6236be6fd4f4f62a4d8794c61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125849.25/warc/5c3ac9a439fd585b3a55672db8b8c9a85351e51f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125849.25/warc/7a78cba8f382fcdd79347be9123c069074d7c184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125849.25/warc/b6fb1f55ee9aaa26202ef641bd299870a79393c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125849.25/warc/bc50bb8164d2ee19f6be8bc4b9247ad81ad38911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125849.25/warc/cd5cd6b63126fa21acd98457b2c6976e47387652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/071d0e186585e54c788277e97edacce5124e3f56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/47b062731ae71f5e4f3595cc71254f157a8bc80a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/505007d58353ead722c29d8e25419d881159710b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/749b3c0a837fd82b41cd0b9f8f9ebfd870e9daa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/aea04819fd2fe51c8739d563f51961edbfdd6c63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/e5fc45a3d47c0a185d01b1a1f9d48f8c8e488b6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126237.56/warc/0233615d17c997c8ad2ffd2937f7567bbd5373e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126237.56/warc/10d0d2fe5c4e2fd1ec8d84e11718ca7e82ee5cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126237.56/warc/1adc6ac77e5d9a4c87da7b6b70b4653e77bdedea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126237.56/warc/a5b4a017d00dff43f68cf0ce68fb8c379c68e734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126237.56/warc/be035c753496c34aea3643e33406800d13735aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/16989b977817c84deecd5f94d2fd78acacb26883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/2e50e401b8395307bbe8235aaee48bfa6a2fc71e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/3430f6932f55f06888dc45ac0638c24377d9f7f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/a89444e00d625923b9858c648d7f3cd8b9f82b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/b779142f6fafb6c7b3199207cee681bdb4e1f3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/b88dff55e87a3723ad962371cc50267f42c08e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.50/warc/0c3556f976b9d9b2adf351977ff9b0893cbdaf7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.50/warc/150e394d54cbf796165ae5b78307d52e0e478e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.50/warc/5f97ac3b8bdb4f08e6e708b8c31f1bc7c56a5a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.50/warc/9b38a1bbb2e01f23d9dbd20a115f37b2493785d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.50/warc/dca46e8bd7a8479b831940c06ebbb9a6d49c2a79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/4057982c5137108927121b87d297a91386e365ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/4856b77cb93c458449035a37ee5827cb4c1f756e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/4fa99677f27395d2f039b164686ce4be42565274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/4fe7ca0a237a60317b759006ad16b882b6c912c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/abe9e0ef529936094b60ee0a832784f899a1b0cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/cec0096e903b7adba6e9cf492795871f516c5090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/14aa2a5020e9aa181e55610a0823fccc8161f344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/174e64b0fa668f8c43a05bcf3cee71cf2c752b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/19f488dc5140452073bfe2e93570730d90eaab0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1a6b333451970c6b2573aee69b58062483a385b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1bc9fbb7c0755e79409b4adf85f56e65ebdcbcfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/20cec85eeb886c0d909a5cfef0c3ce0554aa34d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/2119b29d949b75257456fb886f948df51dca697c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/25e06d818a8f86a1837edc3f72235ad9c7685219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/267801cc66b90ee6c6b5a820bef4b06c226b1147.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/28af8e25c2fce62ffbd6cc17334275400f4a1cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/2c6cb7ed6e05d805a3f8f9fb0bc1c3df8ee2ed40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/2ec988554944d6dd386f073bff9dd38a8f81e52c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/331f1a2aebb4d4572b04c07ee9045dfab91bafde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/39dd05f0986e470bb0eeb296e62d615478124b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/3d842809fcfb319902062ec061aa6bd11934b9f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/42bd6c94aec43a53200fdd49f779cb507b4d30c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/4e6b73db8664dc49cf51f987c569dd3a050e9e98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/4ee5abddbf6af7f14881028954cd5f9abb3d7fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/5415ef074c699faae9334466713146b37ff777d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/593a94903e0f69208e4ff5f898e1f73124e8c771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/6031fb3dcc89f9ed3b45994b351c359998f86a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/6c269d2199e5ac1df7d202598440224e2e3c3283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/739eea540e14f379d22f9565b000d1957eb598ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/75253d0f4a1d62d4337e6a3adc69869718d80dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/7c7bb440503da8e2a604bc0b1565a5cec047f9a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/7d3ef98b88986f5a2f8c11281056ab8b126b6eb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/7e6e6bde0fd64f223f5cb5d55e0815a3ac4a46d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/7eb6dec4968671bf6bdff501857b0173d1ba134b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/846b27b7f4e54374d07d57e1f077f072171a0f44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/87de6beefa432a7075da342ca594056e064bbbfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/8d0d6faa73aa1d9b98cb9ea3b49a5c23a6868f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/8e88789ab2183dac8ec6e589e4a9ca8154f8d432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/8f03b2ff0108066503d21b4de24a03acad75dd71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9073816e8649c688a1d3c97b947b15b096ca40a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/919e2ef68ad940ca073912318ed258bc9c60e1cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/92c58b8c559f442168aa7ebe7060f1352dbb99c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9412299a9359b9dde7e6f94494cbf812505fd869.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9569bad95b2047a8328ce516bccbb752d944a3ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9732fe4e9a85e662218c444893fdea207d9af984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/98b316b9040e670d4685f8985858e6ba455a7c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9b1583065823c591f5b6581c4b1870a822a18c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9ee74fda53b8cb1dab5bba08a82e411ffb2f2c76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a04adcd8a1f3f24f67059d2b4494b883dfea49e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a15ec1427b38e56c5ac849cd4aa89dda4a01ecb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a1735c387d5c18c988e3bf77ffd964ac097b3732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a180882cc29d5212782bca1142ed0a1cb2756f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a41654bc3ce19c189d25139dec2dfbd063f73c13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a41a5c43fb4b45739808586fa45b63262d42ab48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a4de389a8ce4182ae26e033c1f1837cbc67af119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a5ab217399ace3fa353583a35fa2bff59324142e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a5bdfc704d5956dc25724e9a99f593ca3eedd1e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a708526635c153dbd1443e89ae75086e292c0aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a96b2e3270ee7eb13b527cc89c3cfa2b243cbe6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/aca25a5176cb191dff1fafba5620152ea7bfc4b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/af6a319349959b70eec82753a22df05ac2e1f18b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/afb9315e0ba7ba8a7da72ca297e85839893bb4cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/b8bbf527816d0719723ab831b296dbe07c5f302c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bb0e98cbcec75975d1c159a743d3f0013c3b5188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bb7ff0f3b33ed25614bf6f2f5f8a67b691f7b0ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bbb3a6c2e8ef4cd7be78d2164b90e3da6778cd94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bbbf8f6158ade763cb46c0fecab2ad9af738b669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bc26f94c7cede802da33130f87ab02eff0723b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bf6492de0948a7fcb174f28ba2cdcdf3954efd4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c37bba4e088f7fd8b372269938e3e763b35639ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c38d2cd6723f98317edc32017f438c3479a982c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c3e99075aade44ba69df1c3725f85db20da8113a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c5e56fb070461303d266116c878133ddfae74ef2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c78f6c786d2715dbc9c9692aa1d20b63651b4b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c9eb8d848372fcf1758a448f6942105d69e749cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/cfcc16930a38f39d80c2093f067b053b4e1a4b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/d24688eb8fcd4c6e28163916c5375a5968b2151e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/d2beed8709aa181a44c2731fd5cdd01334aff937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/d89d3be5c3233f9bd82c44a8d0bc2296368a5449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/d9f05ea0687a02771d4a0687c082778dbc4e48b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/de77ba3318d66983fd8969c466f2fe01cd4bf253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/e04af5415e8ebd6a0062b8fe5b95d0723c75cd01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/e1135fc50952fcbc2e7bc3f6316a6dff164fd7eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/e2f4adc31dfafb058a1686635abd3fced07335ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/e541db0faed9443ff498fc92b05d4d8b9d0fd07c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/e6ccd85550e62295f70a0d9b58540654f4718fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/eb867234d3e48295e0f205160c4d8010324fbc5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/ef2775bb1af54fd1592d2b9c2a9982645460396b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/eff89cc2ad781ed8e0182b855aaff8369e355d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/f0c0540b55bfbecab75bb2fe9323972a5c3fe89a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/f666e50f7a0ec783357222a42460ec2595e773e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/f8eed6e2ac201c7d95a90c29f1d85b1e5a28af17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/fac51ae0d1573e21611a5a5386d9f6838c5770f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/fc311f44da6e1144cd34dcd3c74a8fafc994af13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/ff823cbd257a3d8a156a38a45ba346c2306d28ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/ffd956964a28ed3ff4732189b29798da99d3d198.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2017-17/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2017-17/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2017-17/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2017-22.yaml b/configs/cccc/dedupe-url/CC-MAIN-2017-22.yaml new file mode 100644 index 00000000..02bdc7a7 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2017-22.yaml @@ -0,0 +1,589 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/01d98e3d2760a8a499fada423142b43e8ffbe6ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/077433e899def293861c8dc7983531acd83d34f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/09ecdcbd7e01649a5f470ab965f1fc3151cc59f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/0d34f7a8199d60678ccf1012df9b0a0d7666a16c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/0ec0c1a6c64e3dd9eea4b43dab3918b4cf63acb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/111235e318d8284bbd4bbc0e142d05a3d96c17e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1197cb8cecb030b1d57a2986ca3d1b18f301e61a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/057e22712491c48eeafc682f27ba4244bff8559a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/6ead7943fb083c9d0c4c32a7a785e7d720d286b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/c0d5af31cf3a33f7786bef75085465506de8c51c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/ca36c37e119d597e63b7020e032d8155fa760ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/caa1ede541a02c006a65ec61bd65b1b75aadc004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/dff6a1c24ffb1c6338dd45cc61fb31cf30bd2564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605485.49/warc/2877a3ff5646c0502d7f845f6ea7b3fe0b305adb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605485.49/warc/5cd9499b011aad5ed8c62cc9c8121df4ece2d54f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605485.49/warc/a80027c42e1bd677263b8084f5dd8bdf5c042aae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605485.49/warc/afed94bfcecc742df8a0a40204ecf14688089f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605485.49/warc/fbe933edda0396823e8e9fefd6a863bacc083b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/34dcb0eecc201f0bedea7b6a7bee8a938ca60ad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/48467a64582ed5426641e0338ede6a72efd42827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/8a69e4b0335bde274983684fc1a64a601062a073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/9fe74eaa59210faba0db436a72049dd670561483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/a5da0ef3a1d6f29d1bf12f764c4e5d7f77bd7306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/ef213702e6a20f45253ce1702d047869786c742f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607120.76/warc/0a901ec287034fe4fb3d86c4ee809bd7bbc8888d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607120.76/warc/157ce22b8f69f31c7b24ceb1cdc6928f64d7de05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607120.76/warc/25d1991ef2ae196aecb212704fd9da4b44e8ccc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607120.76/warc/cc35b7809ae170ed196bbc0f76062863893041d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607120.76/warc/fac28cebb09fe5985b57e14267635c92988cd9aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607242.32/warc/109695eebc20988ccecc88995f89d1e3a06510ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607242.32/warc/351f42d93b00109fbf3045f76581207f479098af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607242.32/warc/723b5c20f245173b46ec313fc6602ed659c6b373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607242.32/warc/a715cede1938125abb1bdf1d16121776eb3b0f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607242.32/warc/d4894b3dbfca401d40ac5761f5e72f5ffa5927d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/499dc89969348cf3bdb290a66ba4e4b722b80b8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/76d2a6e943896cb7231f44ae488559c86d6502c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/8fd1903b58271e6ba45ae9ed9accd6fd91f00166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/c8aa4d693375c193a328f64883d979c9c3eb440b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/eb74b4aecfaad5b2ff9941e46dab2cf1a15c7b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/ffef207b5c2927fb39717467006226c9b29a7193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607325.40/warc/07882e82c07aa5161a611c71ff31eb44ca390e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607325.40/warc/85684574f99e1e0025cf4f115f8bdf051ce69241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607325.40/warc/c39632a16f8a53780b7d555673164672e5f61197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607325.40/warc/c43154850be7f93073263ba264b9a05ba0b5e299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607325.40/warc/c6db036213de305afb49d03fe1a656cc7ac47a6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/3b327fd721dfae27c0f6403f272eb5397a552a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/75e931786eee3e78e74877f01e91c755152f20c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/90d70e728862f1663ad705376fe54c440407e929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/d9c51c7e8931af6a6c43350a7610f3aefc0384be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/e1601810cfa18e5acf1d0b64edf240d16cc41cfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/fe5c636802fbbdfb6427a0bcddd4bbc78019d04d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607591.53/warc/06db369d413f62a2413d97942166ddddd44b0dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607591.53/warc/264b2720b13d356be9b7dfd535a1da66e5de47e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607591.53/warc/7483dde979e1b99387bbcb5c75c95f1d0d7cd3b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607591.53/warc/86070f00b25aecce65f848461e6063444a8381f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607591.53/warc/f82c241cf0afa4cb0e67fd6477d032f37c64d249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607593.1/warc/12ddacf3830aa4a8ee26d68e5549005d89640b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607593.1/warc/c7630853674de2c40572c6ba8c442e4ad379cbfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607593.1/warc/dc1218c9b42e7254e970f7f388a71c877d196d3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607593.1/warc/dcaf01f550734dca421d7dfde630f1bc3472062e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607593.1/warc/e721c94c77bb6659e516f39c603b986cac132792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/0143f6534f2831f35cebcd74e8d6279105e43048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/1b94d19df81d7f28ec482faf0abb864196e6b50f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/96c1fac32622da83b05ace8f188c270ce6ac19ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/aef45dd6586d6f844cd63a63d8f52513976693e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/b3b4f0e303601675709ce53f0ec21cefecef56ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/ccc469e3dac249b9d2f511e6844c9a5b2b61d69e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607636.66/warc/5eebb98665edcf8c0212cb5cde177f55e5c01ca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607636.66/warc/63f796782512c2c115ae12def3a6841be7c1e5d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607636.66/warc/a5d2a738fc4d0cf751992695c6bc3aa5ee7a5c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607636.66/warc/b6a6bc20e0e99116d8a1b424da5c02647b652568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607636.66/warc/db6e4d2c3693cbcbbe1d1449ef22cd66d5c8c9f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/438d54feea96e3b2a71f8ceb6d832841656372de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/70d9370402dab77e201c6ecfaf35be59cbf9e0eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/a09cf24c1fa8a7796317ebc66d2f1919a255166d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/acb506328d963d9c3f8319c616a482fb7c452faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/b07429d34fbc564a84b0024b560c16663a5f2248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/ecd0cbd77e7b4cc5f71a0a1a9e7a3e1026cba486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607648.39/warc/1a5e50b1415c1dd235e290a324913ebd24ac65c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607648.39/warc/332581c816b7f1d6d4661549c3dfcb6a3ea5ad28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607648.39/warc/9a1f3ebd634de430d3f668ca950a50927044c97c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607648.39/warc/c0d1415f88c910a16978d56e771bc4cbe475883f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607648.39/warc/f797eaf8e29df0331910e816fe8db5aa0a7570b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607649.26/warc/17e974a0050d2cb52cc8ed300540eea851f1a0fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607649.26/warc/5a0bb0bc3e1ec71db74f5cb1c5ad65700beacd56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607649.26/warc/8eed52edea3caeb938eb9b4d37e5bcdb72960b4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607649.26/warc/e2c735979f52288ce9d7400ced46c81f71015fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607649.26/warc/fa42a0d14606f37193c02eda227b6ecbe06f5358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/357fa6779bb0bac537ab89693466c192c1f04f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/4fa0f479449a3e46da86c45bbf995a75036a6b9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/73d75c48fac154842bb8658ad6d7967605da7c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/7f8e4b340c2314837fd3cdc37f2db8dcc8f43968.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/aef3066f16812b1c8797d39bdda6587062ac6e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/f1313f46ca43561942296bc16ec240887f3ffad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607704.68/warc/55033f2781a40c3b57403a31f4ec7d72954f7735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607704.68/warc/56894bce5d96ac4878563cc280716a97ae2405e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607704.68/warc/5b25b1b28c8ba3146c3f0c7de8fc9fc4dae8744d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607704.68/warc/9cef2a6d6c750eabf596bbd90f95839c98ccc96a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607704.68/warc/b020431246ab85f9e21804b23535bdea90b1af94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/1354ae16dbe67b122b4315a6ea309b45f0c94ca0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/94843bc620086339cd11f5b9ab72613ff5300b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/9f985022722ae542e5c73ef2620672d96d903795.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/ea7464282387fd4a5e33e39c342e2f441d259af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/ed335cd4994d7d7a2c68c68b5e4fde50f7bee08b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/eec21c1892b722bf7c903564adcb94b5256796ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607731.0/warc/0bd7205fd5131bdf4128f68a69a8244e927b780b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607731.0/warc/2a844dee7b6b945a4e872119b1cbcafe2cf00223.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607731.0/warc/6756f1898ef0f9476502d17ff4ddc57b42245cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607731.0/warc/695d4e14013bc4860bbf0cb49d9348bc9596a12e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607731.0/warc/d975ae7bba37714f409d71dc1b201bf70098c135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607786.59/warc/347ea62e840ca8f7d3fbe133736a55bdc1e3c55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607786.59/warc/407dab28281042624dbb4ed68bcf03023b71c387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607786.59/warc/5adf764fb830fc157882e165528a08c58fd51b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607786.59/warc/6c3973e93c476b0dcafeee629e8522670230a43f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607786.59/warc/a05a2e01310742f9a803d6cc0963825817c632c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/85b00e79d4aa41dc10a9f0cfbb97b67641671921.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/8c0d2481d7b96e3e3bb0d6891da2cf6dddc4a6df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/908fd56194379fa7fc11feae0fed94a694283e7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/b2ec077988a324fb353c6736ded1d0aee9cbf44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/beae0c8ae478efa936429e43faea6655254a7ece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/c1d66eaed2f39ae932d2ab82024995a9f4883b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607806.46/warc/25b4bf747df1c24548b91d9f9b668806db6afc8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607806.46/warc/2cb6f2bba73275261debe859b9831fa125f3869a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607806.46/warc/3f7c9184efa3c8b6a95a80e97a5df711f55827f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607806.46/warc/98f6a2ee39ce2eb96c1908de8709059228866c61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607806.46/warc/f28636aff7aa0faea93ad5213c840f53bae1be73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607811.15/warc/35528e156131b94cc535a60346f57c2a35dc9e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607811.15/warc/3956ae60640bfb2819a3747d2cc9b28d6cc07bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607811.15/warc/841251a06a902917afa0437899c1fcff9c454d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607811.15/warc/d7931d5cdf1865ed8e0c9d281554a88befe69f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607811.15/warc/e497ae2a158a0102793db812d5083ab33a333ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/043a16e29dae8492ac8ea75352148ed8186dfb1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/2be26134652c38c3190bcc67adfc6e058830eaf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/362b88ce038d044d06de33a5e54d89d0cd4fa984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/38af72a8ad8d24c6d9c3c60532c30e84552c336d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/92586b5389206a8d77fcbd5db78650d4d3edc020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/d8786c39957b9ffa3876e9e6f1d212452caaf6f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607846.35/warc/14dae823719bd38793adc6f596c6eecc3800def8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607846.35/warc/1c72ea0d724e5337046e4c52c1ded1767d6a71ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607846.35/warc/5a809ea814eb2327d6fc8279e6fe17c6b16377e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607846.35/warc/708a6004da4d995ae352e74e0b4fbea23c7e6b09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607846.35/warc/f4f0139af8fe467b88a12463390498be95a36d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/21e31f22a0cd58fbbd8a6f0361c6966d811d2bba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/5285d9d3110a38705e7f2808ef0e2cd4941cf018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/baf4e125cc7bd8f42ba1824c679010ca50c3bfea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/c89cea32d7dc9449e720615d6d9af3791bc45138.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/e6be232aee81bb670bc123514be1e0f2522607ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/f8b8c57e880bbc8fa1d21215943b666ea32dce3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607849.21/warc/1ec5bc346176146055a6f8b145f440d3ebb5328c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607849.21/warc/34c2c2a32fd92b04a0b39cf6d706b7a2998cd671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607849.21/warc/3d50221859546db6b6b28245050ac8d8361d432a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607849.21/warc/8f61168ea2395bcb88bb3927fc40e3604199bf1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607849.21/warc/ec307db757b25e61e8e58c7e1da4529ef7eabe37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607860.7/warc/03bee945db9691eddaeb750f855194587dd162e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607860.7/warc/276b0586394c3514b792bb462fd3e7477f013c96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607860.7/warc/63f4958f7d8efd7a0edf2ae4db1a559caf36ad7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607860.7/warc/7bb24950a15a926852b87ee2adb2e69a6a3a30c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607860.7/warc/c652ddd84ef8b5f94884efaf885bddc61bcca01c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/0cdf02f7f1d6bee966c4e8e35f58a0eb71a4febc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/8c141fc602af960bfe3cf8976da197c3724d8abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/c653c8dc7da18cad700c57ff259c2fc59562fe2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/cc3cd7753268110f30338f37e8f727686bd2aa3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/d29a64d4f5ce332fddf63512f3f32e849c7cf3c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/fcfb042f0e2b9b5239e19f0e5ed96e558237ed43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607871.54/warc/36070419297551f15c1da1776258f5fff89c8a2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607871.54/warc/4b534ac1fa757d68f257a0ff1227263d07077ef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607871.54/warc/82e4f5b57e82dbe955cb7650a8cb2d502782e5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607871.54/warc/ce97eaa6c45364657df11c2b78a75bff14e70a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607871.54/warc/e0cfe347a3b7e48cb07f7bfe4b5432ebf85c9643.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/1b3493ec15cb9f445ad7f0a4ad58de36d242922e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/39d5d6488146a688ec4588513fa6167e45a720d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/5b9b54c9610a3619e588a7c8723c8df0ddcde84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/be0ca581e6cf65f5207bf95388a36457b91d21cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/edd47175f3717e72c41df6cd62367596e67f21d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/f0e71636fcda762267138efa85b776de8534f881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607963.70/warc/4155a34744d1c768b6e5e97ecd3471fe4ed0e015.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607963.70/warc/4826f9fe49dca1022896eb5aa5d6be5140423132.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607963.70/warc/4924fed0bdab60d528713c5e9b880e791b954fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607963.70/warc/971950f9ec8f88eecda03d930f44e7ebdf49c7fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607963.70/warc/c3eceb0fd7a8405f9b8886014dc06bfcce2f41b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607998.27/warc/33e80f1a68f3b2973203cb5a5ad33610f5591586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607998.27/warc/6857b3ee35fdf3b1b61e2bf1a59b007b51e17f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607998.27/warc/7154e6df6537333ab3c667b3f8298c1bb027aa60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607998.27/warc/8be92c5aa1fa1dac631df693f7ac49626c72a450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607998.27/warc/bb9c0613aa2fdef0cb8d626191e1ea6d06151aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/4ab258b443b562c784cb5774e3d2f231ae7d78d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/67638f7425b85668a12b3315f9e9f6a3810087d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/7e29f8450811e95d23f7743afcbda37d65e5c675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/acecc14a5766693ef5c747b9e95bb16e247e1bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/cdf48e8d0a0dcb80cc98d7cac3710b8f014ab55b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/d1df63110a0565192c543a3df415ba5b24e92911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.13/warc/7b5a15ee6f12d924797d3ccd1d05a69c7c02840a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.13/warc/9ab59c18ee01a486dd1ca5f665d9562778c8b34e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.13/warc/9c213806ba8d1c4b248e55655f8f2608465c9c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.13/warc/b3cca513fa2ff2469c7479c55b78e1898edbf2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.13/warc/e832ae494b066fe5cb2e77fd3d80de65999618b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/54f183e9c18d9f1f6c70e1fc3b0a051762601069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/5e7745efbe42bf825a88c3770cc45b404b130a8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/8fd3643f42daaca700b6e281aa71b1bb2aa63e6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/9b7df62225498cbd723d3b7b0482d6acf3883633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/ae55e4323522dd1ba9910c5575abf8e6bb295978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/da9c595a661c9556cf7cc42990abd8d81627fd9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608067.23/warc/086496816ab8ff04e1a34c5c75f39328e68f5a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608067.23/warc/54c6e29e48ead3a77ffb928901774cc8826d32d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608067.23/warc/6b53281ac7be8e3c838847bd18efa356a6752255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608067.23/warc/74d31c0e4098b2ec030a0541e8b778ad5c702acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608067.23/warc/953e680b16bd6f2c11edb9f7ae3653f19c118012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608084.63/warc/13b9be15f88f901e937309db610e2ae828898bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608084.63/warc/37e8bf8ddabd513f628c7f5d6cde312cb5a870af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608084.63/warc/7c5980dfce187157a728f890d56ab7e24b74375c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608084.63/warc/8837c8707de4bb19931c323ec84f0a821d3aeb45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608084.63/warc/ac269b23167976417693b6042560e085cdac20e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/29b336d66088cd94a97c29f6097759e7c9ab7323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/59fb16987d3d0065c6db788a8cb4a905dcc27cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/7899c8d449522fd999089faeb38177ce54cfb14e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/8f82a3fcd3448efe14d18ea341d32db91cb88de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/b7b720ff53627eaa14a6517da555bffc64c1f2ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/d08f6c80920111166c4aa3ca5f5f56a847d01222.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608120.92/warc/1b4b608265a1b8e4d923c06248b9f800d29667de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608120.92/warc/7e245685e2e2a9902c2c9689b96cb1bddc19c4a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608120.92/warc/93929aec17ede6b23161936815c4acb7fbb1c186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608120.92/warc/edbfad37408e044b8c8941c765e572a3aa40b112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608120.92/warc/fb8187296996852b3c674f7682d1170b631617a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608416.96/warc/0e60652d7efb35e5e56660f267d5a6e8bf663969.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608416.96/warc/34cd2f5c9c5b65bdcdcd93742380f10ebb89f781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608416.96/warc/7a8e8ef8c352ecc44437d5d867ade14e6e69a733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608416.96/warc/81cafe46d02ca2c72874e2ee6a9f7eeca4d78b8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608416.96/warc/c2b3fe28e6625b1828f9b1192c162664ea8a0f91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/2b8941d3754b363a416ed851023b1eb1967ddfd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/36a6f33e5f6445f1d9f9935dd0e988d72397eeb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/8773411c5aa261d8f7c4b8b50b99a47f46a8fd53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/879167dae1288e248463ffa1a651d73876264715.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/ab9c5cd039c9fdc4604e970f049fb8d406322f75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/abf1bacc87639490f86a8431f55817aac7629524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.80/warc/3e08568723eb349cbfc40030ce37a2d8477300f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.80/warc/4b1067b4fa1d45337245ae807f483e2523fbddce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.80/warc/de8147b6a019b9565cd5b0a807faed15bc740ad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.80/warc/f630706e20f382979c1bed1f78623f6380369b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.80/warc/fb04673a5b6abe888fcb8d7f48285ef9b0ba3e7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/13bdba20679599ad9b58ea912f659ba8ae3c27a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/273107ef03a9e9a24b26266c0fd2f59ad0de6990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/28c8d04925e91b1fd64da53ecdb0bd81b7461f39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/8f66648e02db3bb08911a1d6af4de84459f3d4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/e5b04c823b79b4ac4ee84dc9a1ebee19b39809ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/fd5392876c7ef8429a6788c93e03a1b55ce8ea84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608633.44/warc/81f07d95c05b8c20a14aafa64246bcda0ac32484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608633.44/warc/cac6a989a8ed4e8abb49fe30bde98df04b6df11d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608633.44/warc/d9d8839a8931086929c4c2ed9bfd0f30d05b65d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608633.44/warc/e672e6f916a5d0412a7cd971c5722c4158d1e0d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608642.30/warc/20039d1bcfea1541b05716de2ec41eead269fd77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608642.30/warc/6a0ecf07d219e660340e0d7a735f9a58e3dfb2e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608642.30/warc/c4da8a23ad53a891f84eb4cdc733fb07bfaf41d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608642.30/warc/eeed161e8b952f8c8cc0c0870230e9cc1ac0995c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608648.25/warc/653f8a88e76bdeb31070a29a0f9cdfc6d3e2ccee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608648.25/warc/8354100b17c95e4127c19f3d9b83cf4797b0d42d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608648.25/warc/8442976d69ff7d093eca25516d0a2ef8174083a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608648.25/warc/9df41e5aceb124f1a20e8aff2ba10f6cbd767d32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608652.65/warc/606f330d564abf83629424b9e6fd300b8ba40a83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608652.65/warc/c269899afb08f7a7d9b6872f3054ce6eeecb86c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608652.65/warc/d8eaf1a01c8251542b1ae9aad146db1933f46166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608652.65/warc/e5df6b4d3d9ca3ec7722632164536b3ecaea77ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608659.43/warc/11b1373a158f806361502ff82ea56aacc06052fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608659.43/warc/216096d612d1fab967ab07d34b1cea8c37d12582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608659.43/warc/9f1000e46ac3ad444465cea09ddde6bb6076d93a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608659.43/warc/dbd84985f8901ba14a18e79e85796c73882f3cb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608665.33/warc/0e0ecada9a4aedf7112c0c24c0a9a52dd999af04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608665.33/warc/27b458a6ba1be7b7badaf96ff28e796f0f797bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608665.33/warc/293a629cdfc8efda7068a5b87e838d848f7e1462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608665.33/warc/7d3ac185586dac51769acf7a86ab42951ac31e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608668.51/warc/0c7aec84e41cb4e89a5d4767e67753b25c83684f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608668.51/warc/2cca2627415b64d7f2cdb739a1171edd2ff13bee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608668.51/warc/370707ca5fc834eb24baa0f153afda7d1874cca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608668.51/warc/58bbc570fa4b140e75e24ebc14828817b81ddae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608668.51/warc/bccc04a99eb5140e25c651ab41bd11275f129da4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608669.50/warc/2c8b73f6d29ea83714d7718ba256ef7025133469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608669.50/warc/8cbf56634507c3bfc813291f922edc38b0f14273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608669.50/warc/9099268f09b3b344575fd7377c2301837bc8c810.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608669.50/warc/b5f718b8c81e331e4bcf8e355fa8ca9df177bf59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608669.50/warc/d46f29e632e081061ded1c74feab8a66b770761f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608676.72/warc/0283a8a6197434d6e459d471504f30a334af8b38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608676.72/warc/3756d697518c02bf6b8ec5c87e2acdb8b7dc18cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608676.72/warc/3eeef94074f0fb621f8e0bec8d4f0fa92b400984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608676.72/warc/68ed922b0b2573e0261f2bd27aac6722d57770be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608684.93/warc/0d5d580bf67d7c93198c1182421b74398b733d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608684.93/warc/2cd6e1040f21369732fca95386d71b31adf9816a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608684.93/warc/3c7e5d68182c8fc0a2734c3151d94f21ad998e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608684.93/warc/f111ae13b982698be68e934756a65b7a99b20ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608686.22/warc/65bb969e46a40ce91222e27117a286554dc81044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608686.22/warc/a26538611cc8b03c94018420ce11678dd43f5928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608686.22/warc/ba34ee18e8039e0e78a24f46cad727ad06b78761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608686.22/warc/e35280f0e380e80b2debbfb9d8b3c6796299259c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608726.4/warc/50bae75d24e67198b824b439b4b50eea51baa61f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608726.4/warc/5e323a31d8e079ecc4b27097f29fc6df2eb056c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608726.4/warc/f92c5dc8d9998babe9bc0e0267b3375cb3bd0817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608726.4/warc/fe61d2deb865e323497fe10b8729cae3a2c99c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608765.79/warc/0f5be8eaae4989c3049131c1901e39682a937238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608765.79/warc/13a3d0ee4f7e3f8002288761e16bf07cc8252d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608765.79/warc/405dfd044db85d59b42b9227ab0199230b9d03b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608765.79/warc/e78c58d4e5be0393abdd44b0a47647055ea835fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608773.42/warc/459310f159527d51fee21de495ce311170c103ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608773.42/warc/93b886bbbf17e41132f2e54214f2b2ec78bc7454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608773.42/warc/f3bcb8d244d74098f8fe05bd68400a6668965371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608773.42/warc/f6bb562f730b697aa61179c3592d9fbfe492c712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608870.18/warc/162d8551563ae934dac14a62c72f663032b54aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608870.18/warc/25130c1b516b82381729db39e8a61c83eada7032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608870.18/warc/7b4870360a4b86da714dc1537120aa4df892ba2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608870.18/warc/879cb8dfb0c964eb3fb701171bf6a8257193b43a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608877.60/warc/58209db6f81c691a260576dcb5749140aaca1a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608877.60/warc/5ff1454249c5b5b657c040f997f23d77b98c6a19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608877.60/warc/9f9c74f0b5bac7bd8750ec592f00186c0717d790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608877.60/warc/acfe8b693673342442feae353c660ae323dd85da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608877.60/warc/ca91f318f4f370ecb3250bc61a88f476391233ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608936.94/warc/33d3dc803687097d54da9bb7c1b9b107ae96ea6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608936.94/warc/60235ef7b1a131f4c9a24f541ede6a4bf05b4011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608936.94/warc/9e3bd4274db800154bca0fe353fa611ad7e79d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608936.94/warc/afe01064bbfc13aeb6ba3176d97b00d8e0b6e5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608953.88/warc/83c72f390f36f4fa274695854f1789e238f6f41b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608953.88/warc/a4dc24de7ce8e98f65ed993321397375cd5f0ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608953.88/warc/a6429ad6c1d190e3686cb95a4396626f344e3a33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608953.88/warc/b40d0fb5ddf02ffb418d2b2a87e42e4cd6404622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608954.74/warc/21b99a61339b77588f6beab241e185b820391149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608954.74/warc/57860bd5d3220a3a626ee1ddc3ac194269ddeef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608954.74/warc/7a876e98be0068287ce9ab2ecab2631ab1eba004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608954.74/warc/f6f3d15b4e49d6c84362b0a3dd8e2a597340760b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608956.34/warc/2b8a3a0d6d825e2345fedb332dc467fbfb9ffde6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608956.34/warc/325d45b34a6ad3c70235d25285be63cd1d24df1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608956.34/warc/78fd4bf38a308c42d081a51361bb7ee1a27140a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608956.34/warc/dec4566c6c6b06bbb67bc374de3fdc04d3adc897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608984.81/warc/1a7949dacde817c3d3341297fdc7f2b410ff2bd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608984.81/warc/1f6017c4e13e2ad4d8eedb47ca3f40c9348b3d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608984.81/warc/6f35cd58e9f719bcd6d8aa744efe8e83c488ab99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608984.81/warc/83535ac8f2147cb145ba27aef71099725e2a49dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609054.55/warc/369b74cc54318015c3dc682113d44581a9a0fbfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609054.55/warc/91a238bf1619715b0c621f6e8ec96f0214293c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609054.55/warc/bc7d52694f49e9de7586244b269e58100787d89c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609054.55/warc/e53a34c5de1e3182ab7401efbf85d887735ea50d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609061.61/warc/05d4803640dee7445c9c72857969db3090392286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609061.61/warc/47e7e61fb9b70374157dd4cc8811ee0eedc0cbb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609061.61/warc/c956b180ca2d8d7535df6364b47eb229ddf8d417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609061.61/warc/d74e918457db3373bd4f6d9c8510b2a523842cd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609305.10/warc/3a69f825b63c88c55e1d1e63d7fe3b526ca1a312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609305.10/warc/4db48f538acc4a2bdced6106f50d7f273db94a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609305.10/warc/5e94dad7f5c5b438aafcc4054249cfe3cac19961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609305.10/warc/68551b3d3119c53bbd56decd2ea8ec39cb3e275f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609305.10/warc/69c9d80bff93b647996c14d52a0bfea856ff180e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609404.11/warc/15ca386bf178b2d2512b222812d6629f75360e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609404.11/warc/2ebd5327cf7412b175cfdff2b68062edac9ac409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609404.11/warc/9e870c94d44325764733e88eb490bc4151e6fd65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609404.11/warc/9f039ac6d95335132bc738851d8f0bb7787e3e31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609409.62/warc/3723a8567709d68e35de0039b090edded22a347f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609409.62/warc/8984d0ed9e3fc143e6810d6782cb9161c53e3140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609409.62/warc/ce139fd202cc1e18c84cb4e1a8695c74e74675d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609409.62/warc/e6f6ab5ab0a18dbb8f4789433c2b22f255594541.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609598.5/warc/141ed2b069da03e0d5b326b20a246339d4fe232c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609598.5/warc/b444c35f25db4033a7457c6332d4715f86fdcf21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609598.5/warc/bdfc22b2092d8f90f0ad34e6a00c357222ff31c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609598.5/warc/c1efa7eca610baefe6f65c9139b7099cc32cd04a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609605.31/warc/3b2c5974dc1c466855ecdc753d47da4f1aacb62a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609605.31/warc/701999d40dfbaa2a79cdf28f463e6846639ee406.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609605.31/warc/84123db2e28abb827d176fc7a2d6612d6fde43bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609605.31/warc/af79ff3265892978d39c8daaaf842e71080db761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609610.87/warc/19cd0c27d91a08383a749118102724161e5fff34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609610.87/warc/c1122c0f3d209c9e2cd626fc98177008a46b65a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609610.87/warc/ddc5443b777ea4c8e83edff5a57f4c4b700bcf84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609610.87/warc/f90a56b50f7f5ab2d3eb2369f5f86a058c00496d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609613.73/warc/4373696d2f0d289e6cd3079c5c6f7e5700ba05e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609613.73/warc/5ffe14b393bc904f1bde84695c8574e0012f029a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609613.73/warc/8cde3c61f380f62d720801cf60962d50e9417ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609613.73/warc/c95feaf298c0ebb105c53c3e2bde2610292233e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609817.29/warc/3a37ee94974df751f7e9f040122e6c7c8dc49d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609817.29/warc/866473afa947940ac1bc256625d9a935a2591917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609817.29/warc/ec05704866692d00846786829541feac6db26d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609817.29/warc/f68bb2f35712ed2a21329f7c4d69215864f8fe9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609837.56/warc/2f3a534a93d276ca8fa201c44f3ce527357f1e8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609837.56/warc/b4eb044ca991b6dc54fa74cbdaf31423174e3b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609837.56/warc/c47d1d4ed36bf7e86068f533f76d4b48dc048013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609837.56/warc/dd3aeffede2715c31cb0433c3f44d4304460758b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609837.56/warc/ef1aaf0e880e1586adf607726a3253b38c734b02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610342.84/warc/6da7aeeaefb78ceaf60635a8384607b3ca370955.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610342.84/warc/826fce03be3fa4285a08fa3273950002ddad6851.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610342.84/warc/8821d7fb37afd01056f23e253227cb05bb730b17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610342.84/warc/a7f003e0fc0f8df9cb0e717396b5561194378d30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610342.84/warc/c7c4549e2a16f7c1e10bb1400ba5f41bb0713ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610374.3/warc/17ba68c6f17057ff4e03b7abc586e726b1855731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610374.3/warc/319aa65795cf69728e5720be7a7289c7fbbdab8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610374.3/warc/b4e63beea54ae4f7ce3484b5d1ed53dcf1b5a910.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610374.3/warc/b562af5c0b9d0f49711c0df9ae23d740f6957c6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611560.41/warc/08a02f69513f0dc5de65322f0ed11620cd0355a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611560.41/warc/3dab29659d3550a44128184c2c1efb3f017ef75a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611560.41/warc/6188c3e41522afc2f9247ccdfcb27e4af6c166b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611560.41/warc/7e51a3edcd80ce5ee0f645e4fe2b6c8aca5e9eeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611569.86/warc/05fa973abc0b9c005a05f07ae59d861465b76196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611569.86/warc/5cf70bca5b278b36d71931888dc2d98e1da4b655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611569.86/warc/7dd874f205b7667af9551b155c484dcacfac47d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611569.86/warc/a869f79ad51e6d26113d4b3c25c1ac8c041042e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612003.36/warc/871236220668395ef5b029ffd13f9631c07af047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612003.36/warc/bab4961848c873149afd53bf06ce1de1e42cad04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612003.36/warc/c51e032e46b7506f7b67f6a19c76aeb5f7ffa3df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612003.36/warc/ceb58a02fabe76956fde72770760f89272181aee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612008.48/warc/5eceabef587df2820e003a4862a2d0f5d0501425.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612008.48/warc/741a4a1372e9b21b052ee507c2722cf330c9dee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612008.48/warc/b7a612b0069e83e31bdea79097d114058e89e007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612008.48/warc/f54db7bf6a00bfe01c1a7e84feeb6cd7fe3a1cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612013.52/warc/770d18c6b1801888ec3cca661dd668c01850b5a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612013.52/warc/8a45d5d40f7759a466bed587337205b308bdbc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612013.52/warc/aa02f2c520d5860bfa8cb0b78b8440c7083eb591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612013.52/warc/b44574c931bb52b02f59a948ba292a02227637bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612018.97/warc/4d8d01de3be2a952c1143c2b78fc8c950132c228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612018.97/warc/6059ba218435ace319a21f7ac87cbade56df5baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612018.97/warc/adab83ca78a071f8017f0837bd7e454edb5c0165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612018.97/warc/ef243fda1f28e764be5c23b674245e4e8b28e1f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612036.99/warc/05592b010e06929b8fa1be87ba34ca693c071903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612036.99/warc/602235320571a6e14d5aec0d418d3b00b22bb2cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612036.99/warc/6ae638b54490dc2ab24f6cb70cbf0d0f239fb012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612036.99/warc/a543f3b1bdb9b28b75c9abfe252b45dbe853d172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612036.99/warc/cf00daf89dbf0aad4e737beb0f4eb30c4213c7cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612069.19/warc/0316ea960f21fde4a452a3cd2e0cec4448679022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612069.19/warc/93005d02c05b0282d15dffad1388082bff45468d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612069.19/warc/96467a611551bf61a6f6011c707cf03e7329dbfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612069.19/warc/bbf26c776f1789482f357111a2acee4387d32c51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612283.85/warc/077d10f5540aec470aeefe063f60ceda2dbf8d4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612283.85/warc/77ce516eecd00a521deb5ed9126f91e7d6b12b7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612283.85/warc/909d108ccefa40725614a398b41f9fdc6bc514cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612283.85/warc/cd75f9e59439671bfcb5fd92dc561e42118b4155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612327.8/warc/07d80beab3688ed7e81c274d66e462808070c4c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612327.8/warc/456b621def9a0f6bda32dc2cf2e3e009e516d6b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612327.8/warc/713f55f931c1898b6657a4e581ce7b08f5c1fca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612327.8/warc/ab2578f99a0cd121cad05406a5c023f201ec12a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612399.20/warc/03ab650d680fac7263b129b97d48c70b4464c9dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612399.20/warc/0b6f47635c90f46aff446f35f14e3bb6f6305b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612399.20/warc/11ef56dd25c54eee525a4c8ba89129d4ff1f4a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612399.20/warc/5dcea0fda2d2d8f3caeac4ee16ccdc60dfb21b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612502.45/warc/28d39033adf06acfa970dd675cee76ed5764aad8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612502.45/warc/4295a5cafb0a6718680298a5999e3fbff17f7097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612502.45/warc/dbe62a1dc265b07746a40fde506d04ab6310bb77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612502.45/warc/e233f82328dd40fac5cfd5e66580ce8aea17c01b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612537.91/warc/704e31eaba049c32779c4e8d3e7bb36e3809c989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612537.91/warc/7b4023f41ab796298f4bdc7fff27a138172efabd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612537.91/warc/82928c751e2d45b882dca330916e114bd109c878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612537.91/warc/9d845e8550d3100fbae018488915ca5480793d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612553.95/warc/2b0fdd5de6fbb6629725054d1dbf7de4db682f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612553.95/warc/5989e17d73e65139bb27050279453352de467a1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612553.95/warc/80a3c6d25d0e1de0f83336333df508c9800eeeba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612553.95/warc/c13de9c39a2e1382573c4b9a95ada5a14150fae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613135.2/warc/114355a33c0e0c39785b5734d40965c777a4b4bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613135.2/warc/70013dce351fdb03c3f0399da6dfb454346fc164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613135.2/warc/772dbf48f94bb9f21633f2ae523a4268ef6c2b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613135.2/warc/f7694c4b40bfeaa9a28ef389c4c054bd69539103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613135.2/warc/f868f2e2fc5ae5338280bbb7ee3eaa8dfb3001de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613738.67/warc/6a9a1ca14df9ea6f48be6d8ea5f4ee5918c11c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613738.67/warc/70789e4fe98cdb6b2ac9e212c4453785681544d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613738.67/warc/835d618d2b39386764e91fb24429aab3612a1b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613738.67/warc/8d731d17b1c78099d101c7fa9dd7295280352b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613780.89/warc/141431ab62f6edef3bb4f17d3b004a8d79c407a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613780.89/warc/1aaf459c5482362c45ed1bde65a1615ac73ebc8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613780.89/warc/d91b411d56a84491a7aa63f73ec4b76be22079a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613780.89/warc/f7306a5d36a128df36e03d19bb251a6af6462850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613796.37/warc/2586cded0ee7d5e1f154b75c68cceba33db2cf37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613796.37/warc/3cfcebc7f9caec03381573911064112e79e49c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613796.37/warc/be782c9fa035a27a815afeacb643ae7ee2241bc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613796.37/warc/c190b5a375df3e51a8064f3ab2b2cee58c69a6ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614615.14/warc/73efca1067315c8e51cf11cab7446894ccf7c543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614615.14/warc/988c6d4b545ac4730855b6d375e1fc0af2340a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614615.14/warc/9c5d1a4c4e67cdab75103a1926d712f4f450afc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614615.14/warc/ef0b63bd95e9f7b4c8b50c74ad7dc30028a6c8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614620.98/warc/01b92c50105eedf3ef48e32dc5fa74f05808360f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614620.98/warc/457f87b40c6e8c97568611d088a55aff6e9ea400.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614620.98/warc/65fc3ded6d2af1873e44850e0bffb506ca2d3930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614620.98/warc/b1c5dddb5e6e99650928a00e43542c91181bf3b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615093.77/warc/3d1fb0a8f6b04513b74c411220458e6a88ac68d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615093.77/warc/489466097922bee4e628bb6ddae2f0e37c35c56f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615093.77/warc/55668c3023baa75460155dbb8d2d87e849aecb8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615093.77/warc/c85af6ad70f0c9d013bf0a2d243022addded5438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615105.83/warc/086911ae8e9c3b0b087076222bde5658cb412529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615105.83/warc/16ad54164753e3b1fac01ea96a80e6cfa8c527be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615105.83/warc/ae7c8fc3a5a5b909e40871ed62e4aee383a19566.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615105.83/warc/c967b74e35b5650337a3a7f885c656672ec344c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615105.83/warc/e08e9fb321cc1ed11f3ad40fa29a53d578b3ea5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/16551352dfc578e326609be96f6aa42c92cbc6db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/187580c022d7461fe08a035123ea164e8fcb2654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1941c3c17d9b607ed4be2e76895f3b17089af707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1ce26a79a267e571b3b019d477d394a4a9e96f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1e57c07bf53710dc43f7b40224196802e24c648d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/20cc394fffbea3987a1ea8ac66dcb7ec8b1893a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/28a1f346207071e4b243a9d2751cd1eb35c9caf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/2bcd9b5a8014e98f5ad25a1cb238ea0e6df2ea43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/2cc26110eb2c9fd225ce52d376984499738050b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/2e56f08ac34a29a24926f6c33ade122e268c7554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/3299fb5db8ee39e0916487e79e48909050799de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/39ba4c8eeea16f55a3a4420e2f53e90ec98a7395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/3b68735990d29de0e32103853d26c03e3d77f819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/3bf1ee434dd4b8f641c9a51be52c2a4496dbfaf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/3c047888e83b1ba2c43c28085b12f11ac63fdb1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/3d16df3924b96fceef761178baf37996aacb86af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/455d152a37507d4724adff9b70dd6e8af7ac62ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/4573f892cee9fa7fc8b9a511cc0f5cd88c5716e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/467c85382ac99746fee4d5cf2961cc51a7dcb653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/487f1586f0758e0379e5b07954530768a545bc87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/48de2e34c0b3017bdc6ca035a43b94b5d66f723d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/493a69e71ecad6bb5466b774e87b5dff237f2acd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/4b1785045019cae71427b47fea35663f62c5a9d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/4ec5bc08b23ea26b653912b4db3c30605db555db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/55937bf3a954a57f8447a3e57016b118c9c7c3d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/5790fa206d4fcd6537052d3a7d16d75f08d9c57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/5b34fc56ad123f857d3ebbcfd83afa9a988520a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/5c35fbb857f814f67898b2401ce9d3cc8e2b55e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/5d4a103c50c8e9e0607af4feb388717fdb3ad5b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/5f01fc6a073afd242df720d6955f7b7cdca27eb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/603eb66f62e7359bfccb75889bd7c079338d1951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/647f72e019cf32c6b8b131d8a09425a6b1b50d17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/697513816ee5edc5f31739b9cde9eef20dfb8267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/703589db7f7d19df904cfc4979fed718f88c2859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/726f7904282b36e1f7e5551efd369503ec74479f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/7854aeaa57a06476d9465185b712dac63aad91eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/7bd9e5e61bf8cd02a56e2d7d85da9123e48031f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/7cbf328cea0ff2a0e03cac7b31142b9e832073b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/832a403ef99b8773a9e96b23eebb4fb42a0a5e1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/851e1cffa6d9adfac02f075877eaed91fa47ae94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/85d4ad13099727aa387962ebdcdf022d7d65bb15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/85fe8599bc199a13679db65a7c8972fc4d9c2f35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/871f6c1453912fad7ca785aec460e8fedb060e7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/8905669ab6a03550d51b27dd2c7810bc2958db35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/894c6096ba694fcd379493a7766f0c17ab52a317.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/8b494652b453e58589579250494257db4893433d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/8d6d3453b107ae25cc741f07310eb55790dd67cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/8f6ccc101839d9711cdaf4f4f1d836b1f8acdb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/94b66d1b6a4d7ba2bbb2bb87834e2e7fcc33a797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/973406fb318b3c4000611d4159b68515ad92ac34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/9c920b4743b9f8acad8aa9c76ece63171b4f91d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/9dd40da2c2cf5000453d544889dea99451435e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/a2d053a877f3a5861f04fb1a894c58f359ee7efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/a6850fe24fc91d05f9b210117b550ec6dec84410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/a837fca2bd060da0a6e72ca2fa670c726cb5f748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/a8383892b85a2c4c6184bb805e3bdc2a33a9fad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/a8f8d4a6127e0071b44df44ea2f2b8a62d9f244d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/aa6b694852555903e98ee36fabfcbb7d8b5b634e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/b18a06e7c40a4ad52cba19afb09644e94bf8e8e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/b87046746c8946ae645d236f9a7801e987a88ec8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/bba82e37a264909317b36d20b38dbdf9816d8e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/c18dc0a5aca3e015b3be872d35fd9af7aa6652b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/c38640cf17a394b106fa06cc5805d509b1dd6638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/c737c8b9f204955622b4d64f67896132be144005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/c7f8f4b22b9f5ba5f34ab3971a57318881bba7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/cc7a8fe4492cf6af42f7a1b1c843f667d319f8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/cc9ef93429fc6fb49bab378e24c0d22e8d679179.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d0d9517d0af83f421ef01d083cd55ab921c10c81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d119f8e6d615252578526f5decd38486284b4c88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d56aea0fb7c0fccb9d2c9e5b35e8cfc0bc95896b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d5ca7484912fe2290d412145a2cc1767774f6651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d7c27bbd302252e5e2ec46c310a4f1e547c46289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d969b8930d8b6dccc2cbaa69a150782ccea64114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/da5c975be9013e52016c07e51cac4df6d68e3521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/dea66175bcd6ccdc45647ee07a00454471a5159e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/deafcb907a5abf3fb3aaf90420f11010c30b5a85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/e14789ecc43990fd4cded34a891093afb3ce1ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/e5ea0410ced16af884bea247b3f054660dce50ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/e74715564fbf90cb2ac25a1283d88a4c56ba5fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/e81579ddb8bce8ab7442e5d6b1d00aedffec80e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/eb1ff2e11535b31a6e068ec408b68812256c9aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/ece515231d660251d4b84bb595efde26f946f846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f0d3aefcc6c17d726cb07b6807a85e9573efc70b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f2af9649d32b5c06562a6fe6f3f93e105a494df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f495400fe28c77696c1ac77d027e9ec28363ce54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f6cb4fc9855e3e9a1ab8abe82dcbb054b0173615.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f7e9dbc7a3a1a761b83e5f0717ee26e5587d277f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f9dc4eca32463c2a3cec97cdd43a4fa0c4d17455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/fc79d806bfc9f48b3da5bf500a54a37a42ecedc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/fd8de760a0da5d2d05368891f77a1503be198024.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2017-22/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2017-22/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2017-22/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2017-26.yaml b/configs/cccc/dedupe-url/CC-MAIN-2017-26.yaml new file mode 100644 index 00000000..f807f78b --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2017-26.yaml @@ -0,0 +1,740 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/0535dafecc37b0b4820efed50dd6e32c67c4c64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/0c0636a46fb29a37d22135a9a1d86a0e4dc1d8a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/0d87cb7295ea06d582fa6ca90c1e718c0ca03216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/11d2c7e18cdb761f04ee7d257a5215a91d249bad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1358d7273c4d7b2dd1311ed8db5edd32f480b504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1467f82a39374bf00cbdc535553222df2cefdbde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/0ac279ced849afc090b0de991eeb44b223b9ccad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/21bf102ade683d32a6bb29a56253a76918dfbf8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/4d49359ce63cbf470753781a31157b92aacded8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/673e1c8ea4997c143df3f82c467187ac31203d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/c64611d00454b0a4b2a980db42c05d7aec1736e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/efb00c5819775eabf6ea9922b3a63e5281bfd91f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319575.19/warc/31901595ccfbcd06d85f4a596289340ad55ba7cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319575.19/warc/364db914c86dc3e5eddd9e27272ad538c8bbeebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319575.19/warc/59c15f0bca8414ddd66d4530872ba5d646894ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319575.19/warc/f06db7ac16024249582f09e2c65f50a561ed6adf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319575.19/warc/f5e450ceb7acb5ef635854b2148a56e3790c0959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/4f39606ca01a8543eab90b6c697a18fd8e2b2b3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/4f5eff497f7cc3b64d1599fc4458311ffb4548aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/5c36ad6e431c89b49f7e6e90f519d4a4733adab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/6755959ef5f781fe98a102d0c49a611b1717d568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/800c436ace44370c02b03468e6b88ef99ec1b74d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/8f723653f755716e3d0ce20486173d5cf3c5765d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/cb7f1457a0266fe8c697e4d2fa0af3c363ec5e6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/1e06c141177d4ddbfc09dcbf30dcffeadc98854f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/2267f22d99fd74daebf07f09a426c0040768ab4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/3c0cef0768e672d1fc0cb317d1cf83f1627776d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/4efe06438a4293972d3902b606a55c73512704f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/50c500482213deaa8e655bfe902f291707537535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/53e7cc2de40b1179df5161156e53fa93ace25e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/5542282d68754821f6abb6faaa51899343ce884e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/17ed111567093740f6889422d170566ffcab8ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/2294867d0df53f4ae52a6c4b1a81c806dfd20002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/48bb3d6058babe361f7a7a7bc0668bd703c73082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/6ee80684b5ce789ffc478a5b11bc38c8dc9f42ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/b70b320347a6d7a017aa083d845bc186a001f938.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/fb719719458235968d584d224e92c1d3c348500a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/21745c30a8de9e83c5dde43696f0db74cab9abb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/2499e3b323beb3d8bfb46f054b869078b45021ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/691d1910c43c167500cf9e26b72a572ea694ad4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/7ab0259242767da44da9084c99c7bf5dc5084bf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/b04ea60ec6a5daecee766b594fdee501dccc667b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/dedfde48e3f4b6ca6823fff9452da0b2972ca5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/0caea6654602a43364c379c879ad106f22ad5c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/5fdecd8d023517140878ba8f483daab2e96e665c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/687f580ddf56f67a30706930f0b80504534d3194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/ba0b9fcedc3b3e00acab973905722fff427ac31e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/eb4b17e10e1cfffd5ace8f2e1c36ef8722471e7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/f6ac7680231a2bbefc6574cead6b2d9005042d93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/083f3e30567ee0ec7fd6378a224af5e669c7aa9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/2632703d65ca924981a1bbcbf964c80f46ce82f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/566272a5e38ac377690c6f45cd1f22904e8d7585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/653d7d013d441267b2ac4d2e50c7f85346839f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/657d9b7f93ae94bfe30389f43e1caf2a33ead617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/773c79dec6f4d6446c6447ffa2c09582a0fa3918.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/ca26d0edc77d58f4937628495fb6dbb417998733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/00dda92a0f6bb22b8c146aa80e168fed9ea431c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/0386e61cca4820aa09eb157f929d1a25e1a9a70a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/1e98a8ef1c3cc4c2d50c5a4ce38c08ce44cc1781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/7a004f5681857571afd231c0fac02cc56cc818b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/9910f2956cb6b5ed1214e74fa3db52e39f018274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/dbca9827f4e2ae75ca996a7516caa8f04be7bac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/ea7bc1cf2bf0886311394ab5d13e1ccc3261b711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/082350aae3aff79ef6e1d9a6a5c56d15a3337b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/1144e9ca6ac89c9ce690930f8b440ec2c5f5133c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/1e842525a72c1e99ea3f664f6a9314080cbba174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/4459ab52ffb6ea1da14d5c65388eb3e71be3a91c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/8a5216183c7f8aa35d2b75224daf1490fb26c719.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/917d955f4f0cbaf90de6492a0fc364f99b89ef3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/3d11a13f85e1d3eedb44817a01fe1dec586cb3c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/51164d697b281ef1278a5ca1597ab58e47e77985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/74e92935225e808920a9a368d62d80de37fe69c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/826264ba50aa2b063a1d2f2b59d29d0530de6ca4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/b6aceb3a634b700552e804759aa0ed8dec42fbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/e65c6b96bb21395684d51e9b04fb8689b509df07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/1cf44b6337626475111ee46af5bd98afb5d4dbbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/2883228f48e8bd524699c62df49d039c81acc18e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/55c477f816c602283b3a3c14b8026c247721e62a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/591cab9c73de7b4eab3d89b7c762ccc60a4652be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/695ca5bd86b8e2a5598c456bc814a518a0a3aebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/c1def38b258c34188d2e89e6c6a121f4981c4aaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/14f4b276d8a0d312cfaa82e9a807c582125fac40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/22044625f48b277c034338e617da7cbfc80a6569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/645c7fd4ea7dffd99bd106d2071361d97b9c8320.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/d16b9af1d39e8addddf5f9eed2fa406e7e83b18d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/e8bc024411dd2a5bb20aa66d587a236e490794b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/ecdd1270b7f123a917550adad71151914e174a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/fb84218638df71db55c80aa9c0041b4c135a6586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/5dd722f012a0563794eaa7144a085eed8f4fc9d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/9bea1610e6db0feeaa0ef712702b9ada1f1acfb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/a2ab64c6f6a37cc3450713475f981dc999856b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/a4720d9bdd2e67f7f1e0ad4979f9f540b01f41e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/b3acc363db67c03c457bd4818ffc28d30da4cd61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/cd5914ed9d02c9381bb36d6ddc76d1086216dde3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/d33035576ed99b19353ec964faa297ef468ce820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/277e481fc2c03ed0f69d72012350794b2a3117c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/3b026e5615d4d1c277e76972d917bff257b67cad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/44736b5558e95caf02fb6fcbdd702bf840e63d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/781b2b180ec497bcfcfcd972ec7c832dcad0fc9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/87cef39fc116b14304b2a9fb23cc77e71ddb7316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/ef3333b42b8540b163eabfc80f6652b66a1caa6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/0c4e6248ab46a455500fadf1f1aa097bf726b84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/0d478e9e5180d760a6b3bc337039becad86ca7e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/3b99d9d37517f67afbf0d6fc8a2b8427019d81f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/5818b7f90257c13ec42c37ec9667f672e55cd3e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/6dac6a93a45223ec0fd664050e951b03b1ad251d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/d824f5346043253e17d48cf161db18b66c093b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/1df2265bc9aa8c9be7baea95ecb3d944cfe9265f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/26f05ceecb93e77bf591a5c286274e65fd44ff66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/814f15ff780a47f199080a0c22154180405421f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/a2c7f7235bb43999db6e484853064626fe1bdc86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/a694f1733ccdd0871c1d394ebce9b7e4df4cd4ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/f97d17cdf297c07810260ca3256faac47dc12911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/01b1434d77fb6925231250ac2b1136d18e09a608.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/36489c86af84d6095c09d24a3da198bd34d48eb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/6c7ec817b71902d0eee2aec6467191250ec9d181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/6d842e82ce0e8465cd1b485c0646eac103062f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/94890c0fb6d4fe06a8609614657e06e6b41b568c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/c73829658a8078ad57b74c5381743f832f19e3e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/3b8f37c961deb275920d3038d24ea387bf6312a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/79f814b0e849b73de6ae013a72113a4c2e7c398b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/a0c540fbb7dfa241dd8185416bdf4ac4daefedd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/ab18c9f2ba808ed3ba62c4277bbffc79f62e3a67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/af236467adb5da05229580164e1293140d9d469c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/c738ec6336f7ae59726b5ebc9c95b93dfe65193b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/cc0eeb832a5d9ea75bca823e4e5bacd8b625ef82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/1b58bb41d9b85e59fd2e1fdf118ab9379df181c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/32bbdb777bc73c039e7d713d29bd950c80a29fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/4e567a3be8819550881c6d5421b69e035c59fcd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/8688d98a5b026acb439d8629041cb2a3b490e295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/bd320f278495beebd93d820a39d3d0f26d8f614e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/e79ccdd306891d3facc81e4ff6bba3f2e7ec03a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/1ed94b1c42fea94400b8d1ab604acf5cb88813ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/9bc4e325096c64a6a6b79d32f0396029ffd37449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/a79e02f5e7eb985625f16272370aae37ca9826ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/c9eb4be502c3dafe0e1539894d16d50a524da236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/e5b7be304932d14ed7e50a5f9c3fc46db15ba75f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/ea43646d76df188e7a04cddb7f23484cc1230de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/5a8fd2741cc230f15d5f1973c01bd3971ed8cf4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/6756a356df0b2eca14a4349041a19a2b2b7ed418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/784ba7bcacf4f6d5caa1fc0e616b1803b7650d2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/96b153b087d76a5a955014975db6937514294a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/979e3978e6e07002ccc028f7d8d7c007159f20e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/d0d8bbf212cfae8c0d4dc2b1e404a81e01b85873.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/29d9371ab47fe5a1b0d827504b59f627607d155d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/77d02b55f85289a4d5cd40750fe59b6fa6aca63f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/84eec89ddd9a14f45be5f699fc23db9e469fa945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/a9c4eb88342b6333546df4667bba985c32a454f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/b8f523b9377d89b1f13c6c472d1905fad2896e64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/cda5080ac12d22bd3a25d9683893ea8e7b6dc1ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/204a3bb8ad04c67c1b32785148d899bc0e7c862f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/bae5b1e26c868c3ff6f92d0efeb5e4289f80a221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/c3586be82c877695e8e202f8fc805df277827e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/c9bb94c00c0d64dde6c7fcead99505c465b39a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/d25f7b3b217a576e9b9067a0b237d9e278017955.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/e0760e9520b4e2883ed7b87766fba60102228e74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/e68c49ca4bcb5902e02f08db6ba5b0f8d46d8a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/50f65e7d165935e157176e85290651f8c27c9502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/5cdba191f8098df280754c4a7a5b5698e9b581d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/7ab58ba8dd90696e78a9daf298f90f7a63942f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/84104c2f93494c19c55c392605d0b41fd3fbba6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/ed85a550efff7c6ddec1d03f1177571dbc28c5ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/fcb86b63a76fd0b857c1bea0ccad0a2f9ac39fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/63fbcb9359251d9efa61ea2eac7ca9f8fbce9ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/785b4f7196f0fa1a6f6ab2b3e8932c788b29859d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/a19b796e580bedbdff4898debbb479ccfbccb040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/a8594ac1a579fc647f11fda7b96f352440017c1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/b767a94cff9954f4794720a6cfac32c2511f6e2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/bbbdfbcfcfbb757bb497568d09f69708696ec47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/4fbb71bed84a8707f62bfd1b3af398f6d39eeb5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/5151e4b6893f6b36ce3ff9e0b3c1731ad5e7ca87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/5b20a5e6426f7f02f5b65bb9bc5aabaf75e612dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/90140eb1429adcd242709c4637a80d2c27c6cf35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/9ccd1c25ee88cd60e0f8002281a225a80c4b6e42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/9e28a43908438e0ff3d97082929c83da85e2c823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/08eefc47e30cfe3272614b64cfa3fca0c7c92817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/0f6c9481de8dc0003b7a9b37414c4fa08a45662c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/740a281b69ffb3cf152ed6625541724e4be103de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/82e2420d9ad72398bfcf95c1ba04c4f65b562ebd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/b401a73e693e60574383efe64707fd0193dfdb41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/e4bd4cc0bc5380af597ce372ba4b3d20f49d1451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/11af81b27c82311f7b2e697de4365edfa1479cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/216dcc595694c3becc984e61be4a9253b7a43cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/2b73991c3db0e8b4b3791b91406371044e5efb23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/38048b09d9bbf6622ea74573ef05f0c698ebe4b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/743bb5f62245f758478ee906e9a5a5baed30e50b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/8975c630366929f2e71dc50e3afb2dac9f0c1032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/99507c5290d2ccab5e66dce5613aa4ff6ed4bf56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/1e638150e5e5ec8182c3ae26aeb496081556b450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/1f83346da1f0c65121e17a2367147b483466b517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/484c0fa4e95b4d0a8a297f414093481dc5b13e15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/69a0334f5d547c2985b83d77d3b7be51b33a416d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/f962432379164ab3073ef8f7ba754e730d4d1856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/fc0b78f534d01524231c9b014a74ae05e8b048ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/589de79c0abdd580a9ef1090416fb4a1c8916510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/5c606bdc1bd4391a31823951a8c684c645299c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/8093fc522cb599f7849fbaf64533e3fc9e539a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/88954a116dfa6f23aa9429cb992f1ecb3f7007e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/91bee9bb131be1a7ef6d33a901dc0ffcd977c64d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/cb32f3512faff8ae89790fae00703c71f25f7f1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/0fc9a93323d03171090608bc873d0a5efc547027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/574097a63ef8983f482669bf79cd3776356b3d81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/844b05ed6cd7ba43101d2779a86a71127738382d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/a58cc57e68f61b68bf174ad2dae964b081825f4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/ccb251e3e0fad63a311766c5128a77ac6f662df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/cea05a6cfc6268581105590377ab561afac2782f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/47981d353ba4eeb244e396600e115d5588d68d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/4ac1eb39b2cb032ebf2d271b2e8251e824a93436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/4b092832386fe6ed0b61c6a8cb03ccc9d56c6fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/527b944594cb639f6cf7a75879a007e8def3744c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/a6f6d6909a04f5e856865d6484cccbc9a51cf604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/e9d98e54f33d32ce63dea896018f957a1b14fee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/119a063f50aeef52509d8fba6701f314e59ffa76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/157432f41f958d705a226311aaf8a78e31c0f580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/23348f276ed507add382ca6f609107ca6ebe8d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/4e5e64b2573088da33ac3c5e634831b71d9fa589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/5b1bb7b1061e9a656a06e41c106ee88499f80697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/877062f01263bfb8b2b24958c03879cd84e74b6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/c981d7bc17aa252a076dbfb47fc95968f73d475f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/24894c047435d1c01c14529b0726b8a9285ea3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/475a8b6b54e5245e2eb095ebafba2f5a5a048df2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/5bba8668ea97de765670203038df4583f2e848a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/a7197fe7ee78b7f51d1cb826d9bcc7b8966768bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/dbcda616f3fbaebccef4fa51365bb9db7dc6bad3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/fbd4d6de4612dc6395ed2ccb610845c60a9ed49d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/969c38c7c8d1de5cad6bf2bad5321108a1e56870.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/97a29385d13d72bad7e58992addeab4f97bad114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/b63882e1275868260e18cd7ce74f56c7436294e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/c37d92491d79e9c9af47391e23e0e3dea3f98458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/dcc7afa8c022c16f9bbfe85043eeb5f2ff0b0145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/f5d94c17d5b49668eaa00293f53e36e9f8dcf96a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/1388493cc9c6654e6bcb3de819acf7f4e2751fce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/181f8e0387bff5304629adb7551abc32dd1fee9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/3314bb0418765a7af226e80423189b0696a4c147.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/6c84473ab841d2704edf63fef60ef27bf4fcccd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/75848e398185bec7bedabd988fadb5da1ce56b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/9ffe1ac4d536532ee0d90549ab6e296c72977b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/22a52de1f0351aaa67f596cf95985b297a9976a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/2943385454e5f173154971a2cd18be4e7689c688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/81e3e88a39d2c41eb16b424c15a57d4f42a1d8c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/c565f0d50604fe2c64843c0714eff93387286516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/cd26939c77efc8ab937a167c5d3fc262016a9e8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/ddc723c364b9a4617234297ce3f282199e6623ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/5696eff20d5286fc6c85a8e57dfb33c98a3982eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/576392bc3b9879bff5986275458d31dfa28bdd92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/5fa5a34709ebc9dcd59dc579d711f15b480689cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/88f37ea10a4a2d0ded382d8d639eaeaa2b4e54e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/8f7990aa4ec0af37c4b287370245847ae12f0cff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/be1c1dcfbce1aafd414ba7422dddcd65c35ce295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/c967c54ea3e012ad42e563c5c4664a5c8dcebaf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/2f5e41c222cad2dbad174702207cf23f9d029068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/3b6e9e86ffe58b415d9e6ad9e92834906c09abb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/3f667cabac34fae25e3685fea5a3dedb516c61ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/a338e7b2bd0f578688eacd49dfc46bf84649b2a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/ca4f6dae35239696b7d3731e207164ae212a59d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/d5038abccc7179c735f1631ccfbd1bb965d03be2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/3246e53041ff46e287b90ad30e68e1143d32f453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/4ca3167ac2a2a5534f8a1185a2e82094d409a46d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/a40f2f6ead66a5c3e859dcb33f3611cc0df291d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/e62a4f8f8bf7ef54c66553be3351a83b3586fda9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/f53fc250afe2656080d67f8ea4d1ea0d666080e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/f9cc81b92cd3629b2827e1e0d90495f38343cea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/11c67a1354b74a56fceeac586de6edcacc5a79d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/1b5bf58dd73c5577ac196c9f6ae82982db26b186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/3248fd059f21e5851668c9cd4ff3faca8413af11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/60f7bd2e0d9802822542edcbc1edf172295f5c48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/7b15dd3d175e66b5075eeaa2422cf2385e756be4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/9ad91b4530157873b5ee459933182f0c8bf34ac3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/182a816919e24b7745a76bc6a29b2d285a33e374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/2963fb9a9b9cada853015165cb59a36ea2033294.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/3dcce4e567254760be8628ae4c31d6bbb1c549d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/6818cebc8ddc308f8e4edc74f0daa049edeafe5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/6e04ae60c0340f6d016f74a4773526a9966be4f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/b9920c89cc0f066b3588c7feb2c25c27f7d58a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/0d2fe43d5c1483ddc2cec29982da824c4dc0f459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/1afb73932ce06ab28ce2bd9c97849fee0e3f2f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/45b049977250a2265c82606a7ada0c7ec5981b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/50ce6bb70f453647c92a02b0c2debbb034b9f3e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/a656ca8b3a07924038eeb155ed78446cc32a5aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/c252e20c08502ae12f153cecd92e9d308eab6cd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/cde3cedb2e5127400088b3b778c28dce33b5db7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/15217ffbee3ef425a1db021c0669b0bf21640489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/184d8ed9a1c9c5053869d1c469f5ee25503a1744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/795ae999537a06b3b2851a6b5a0413de627ca970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/86b7be2547aa1f033f9eeb33ef863e5a4c019a58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/cfd069c28b4d44f09ccd0a14cdcd4be53d1ede49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/e09fae44212ac7401c2cb55fce13c9f6c55bee4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/532f1b674b7b74b7328492e4b6f071abb513f482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/56366356edec0ef6e21eaf1a5e7729288f890659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/66d157f7c54d675a83eb48c198124255a774fb9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/6bf43bb338186c8f5b67030ccba26ce1ef9a5436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/7545bee40c7266de51f42fb2f39b3a0662981296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/cd4a7511147ad794a51b506b0ffaaaf5288f29c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/07cb1025f41c1d4888eb85f92405aa1331143c44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/2b0b3ed3cffe5217c486902bbfbda028df838e89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/2d71798d2a56d7aa805ad1e86aa1455adc6f6c21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/89f93e145bd0daf68d788c6da674d682a667bbb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/b485b10a176ccb335ff115a088dcbd451bd49c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/b7866c25113e11bd3d1df7551c1c675f0be3b490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/059b9f6232c7482173bb36fec9ffcee5cec5c085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/3f05c437abf266dbbbdc0bd24e5fcb9503a59c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/7d1c0a47171f5a287b90defc93cd8715f7265661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/8a4cacfddddbdacfe669e368f44cb073d1db3ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/b15c78f92670eb4635af379b87dc08c08dc109c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/c0329faa3b5c5ef7a321d12448cb184ac90c4a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/00ccc38484a64d433b10da4cdd7bbef24d9b79a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/084a6ec1f3f8d346aa8d16389d959348e9f13e0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/2b7b2b75e01a4bedf33abd3b79ce1397049f535d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/2fc292f5d5828e0003d40ef375996ba24c8f7ec4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/6ed41abb419a9479d4ded59ae730e23480676e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/e3dcdc0661e70f4a51379c98e7185988e0f4fc21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/ed9e7f7f93bacd4c0cbff49978e087ef0f46ff24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/2a50b99289a9c17c50cae2bbbe0c626e5a08740e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/702316d21d5b5fb419247b55e26c6a2216fbee4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/747e2c08a45020185ded543a0638d5b1052222a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/a07d2cc0ddb81e8e02546883394e360465b1d17d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/c18ac0a9816db1ed31ad8acd7cec6ffb16f7761e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/e43656d275fae87bc6c1f905b26a650de1dc4dbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/16d0f1135ff2c009ba1346d4515eeba1a29ec21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/315374ba5276adfc092d56d72885cb8b57418291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/7bb0af9be93e95efff860c641ad22c1668606576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/8c9be7718f3f9f06660962ba63c17cf54233b14b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/93effab152f2fb5644c616a9e9e67344b837ce03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/b8870c36d527246e4a858cae2bdeb09d8eae6d7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/1e9f7ace36a14f6e47e6223ff558d945415418a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/276daa6cbdae3e81e5df5a616cf400734c584350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/2badbef416cf562668d007143ee2128d42f44ba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/459a5d75629ab230c9170fea96056ea392405545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/a357b9699d8646ad1bd86d99f685df7d8661600e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/c53bc12c65c9c5d2ddd09aadc689546d6c23fcca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/2061b9250c7d43dc203b9cd238f45497ce315c2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/46953124d6802124e624370e93e802fcccd8f182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/488e93797c1f5e4fe8f14d118871bcf4e2f625e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/7be180653f894f0641f8ebe6b0226de61e19cf78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/80cd87f1942ca551c28e275de2c45f2ab6fe25f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/e0dacea3ec7e1a718de71be0c6e4d10a74598994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/0670a7dabf584a3184a8cabb3fef64c242be6665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/53771b33bbd5c4d2d430532ac8b85af254d1ec83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/623264eb19e45f911020ceb300e0eed30f08f619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/9d54a8844bba618778978c7085c43908945827d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/a747e26e8f4248f1c32a686c18488ac2f3ffc996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/e381001166760d863b279609c03e6a8843da4a1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/fa3414b304c2d532d1508bc68d3c354621700efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/2e7b0bbf76c1ca721017e14b117cbc3e4730787b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/3b42e1f3c3ffc8a7d2e5eae308e0405407e48c07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/6e5b494caf723f384e2192d926bb6b8ca49e83f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/a94fd73ad4f7b6b32ef7873c3e5bd59da69f955c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/b3300f39fed804b625fbbdfc986da48cf60625e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/c3ffc774151183eeee9050dd209e332ee0ec018a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/2a77c0ec80e858239389c2e6a8dcd0c1fb4aa85d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/5180f446415740ff1556b89548d4db6fcbdf0418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/6b34c0f6eecf34cea8fb475cd9b2a2340de97b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/72bedb75f8e1bd9e906f31921b67492a93d1c4ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/9d6384b1db88fcf91b66481ec12ce82940f9fd7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/c2ea23f9d2df752d924c0579b503ef1964dec02c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/38d94413e8d077e57be6bc358757da645067c6c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/39599ce01f4d7712de51fdef12cb9e9d97985268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/71eda191e36330ac39a52e8340b8a874961476d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/7efe72d8b49b9f5387b359d477cf07029174da4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/a7e7a306e8b0f8a973dcb8bcd11236e412c2fec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/ce78958b5f618c88476a98272200896683ae1907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/078e91e7ba421d9e519e2eb8fe245646a13ef297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/13e7fa3f6065fc2e0b2662585e1810d372d64b9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/5cab6bf3d1fe9fe73327cd878195c39e171aadc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/c88bb9e60261b98869264f4ad14d64e6994f1699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/fb12c2df25b912957391321e4ccb20d68cf006af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/fbbf87ca1a8b2e14792f58e10f3a97ff9355fa6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/29984a8c991cefb8596a971cbec6bc054800852e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/58ac9795f25e56972d46b2d323564c37953c75b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/a691dcef9a2f404d6637d61e4dda4a5df7b4ba4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/b13b3ff1d898fb8ba31dd4c29ce2a7219338fcef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/e6fd52cc94cc90491c531e9f2b87bcdbdeadb545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/ea2f0d0e2d905639a6e00e105c84c011b8fa399c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/f6bfebea8acf42c352bfb24fd184ca413be05085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/1d57853bdef2f1e95d78368211d90d49bb009023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/25401f0b884922ab6de893fb2998ed622ab0f9c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/407e2f4540b47d3e03427254c6d4b8ab80615ff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/7b5ec802d9400a48365526eb38453e0eaad75466.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/a5f4c54ba4f69bf8add84332732380fc525150aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/f27c8c95710a9e67687d6a9e9997cafb0a63b743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/05a12de2eac32782d590e727246dd0d8e69a5ed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/9561560546874d2abfc86bf128c4a29d856e9231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/b1d650365c9ce9b0a77710c97d9470e81e608e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/d11c0c38f1686f1c58aaa22939e179f1a1a6cfa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/d38eaf6e0d4b173e7bdba170227e91d3851500c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/de1c035f3806e06b3b40a0f0d5aed371e7e9a03c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/2563c212b07fa2a29c8495d3e0bee01afaef10e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/2bcebda14eccb4377a5c6fcf144027bb1fa3693b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/871c7a908d8766a4f3892e3e5e94112bb82aa481.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/ae8cb4e24a715391b0cab2f191bf1d2255466fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/cd18352c143649087c24cee10a75f8f043ebb735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/d145103edead3397ba200413924c1cbb57e9f8fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/32c4a02a4762d58115a279b5bfc6df825475f58c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/52e2887215387ec052b37eaaff066e3a47d9dcfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/566c613a247e54a3d1c3236b421b5e77d18a2859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/57f556f317594f2ce84dac816624ca1ed7ff4337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/6c4e6ee1d29009d6158c8b99fd13661561654d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/7fd04350fbb21c850d7cb440e887a3e5f0b4832c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/06e376843dc938c7e904a404fbc258c5b8b7e4ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/1d59712669ab0a84bd12f99b1e7752d53ccd7bec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/2b927511833f181beeebe0c962a91d1c5bcea619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/367bb512459f32a03c56f7166eee4bd58ebd35b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/6a3c31420e5869cfacf15da201d43a4914dae788.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/840b8ffbddf6244e0a0bad325d081025d97b3896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/de813e6235fb765caed2585b66bd2b12d9e81e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/16d081a5c538e0e8efca35d0d8aa50b517a8585a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/1e8d2acf00f29b9604596e6ba08ca851ace10325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/3e016292f92ad017d838b75cabe15aca133291ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/5f550e919ee4fe73906896b56ef650a40ccdeeb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/b83121a885bb1ee5b43791dcf067bf1a44b61807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/f6e8e8b78362522c20178911963ba6732dcbc1eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/48edc134c6cd0187c3ffaf190dddf5e40b0ba2d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/4aaa0db61c8e61ee26d432905cd6d405ce013e06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/b89284929477ce8c0dc901c8a7bfe8565055639b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/ce242f4f015d9107ec2ba2cbba7f361117c70c94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/d97652a0854a067f9503fdb1920a0027470c2bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/f708129aa2f6b96a280263d22eb25a23240a2956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/66b142a7b668172a70c470c722b646f3a8516620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/6ca0b83196a1e32448d1fb2404f5ff4a032766f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/9b75ea0af68229143fc0737bee24d8c4e928d2ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/cff2362cbad26fcce24c927cbb17c83bc74d3c6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/d4c2df36cfafead0849c9cadeabf83f6cff23845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/e36f95594757ca0f1998b36ced2d257be777b73a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/0c1b21e686d4034d01262d0f8e1a71877bae2284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/3815976d3e12581e8d41cfdf1cad9ca25d5c96ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/3b87ea9fa3e2c265f49b084d2189cc25b5e151a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/3ed915d3714b04321f09db37513b6540adbcfb5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/cfb79cfa0ec2af14a13a386d9cbc4dab1267ca2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/e025930ce6b2269698de75757b23cc1f6bce1cdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/03350ad1ccbfbc23f7766e55deaeeab30f78fae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/2548a79cff6baf74f9a9498b9bfedaa50a7eeaf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/6d836d0225b045c4bc23a1b311c20144d6b20191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/95162debb3b76a6775af6bb1d6494b7e0a5a34f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/a086879b562937f892d89e456ca1dba91455328b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/d9d9228381dd321c9ad9a0b7e42d03bc56ebd7fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/ec2a5f5d71915ac0fec1af906c64520fbeb89ae6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/3d0d4b684b790d92d7ef7ab51d756d4da74a26e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/71666491c6e3e9e03fc95eeb74747c81e4732e94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/b3fe4742f86c172239aabac9493ef2a5ba2dfcdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/b684916f0b15128766f0526a6efa959673be7f87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/bd51368b9913341111bf21bb272851477fbd77d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/f4dae174118ef07c0ee2847f99f7e7a49cf26943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/52b95ac78ef551619027758c37c9bb0bdab7c230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/5effc9e45defc62ce4dd323057650af2e2af6b5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/6705644dad3783eb476ebfd2b90f03ed40560d0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/676c8201b23a1eb229f8c24c0037316836247be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/c4db8e3cd5f2147ee65a28405e6d0cfcee75dbfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/d6253be3e853d0cfd7e4dff6755537469b761fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/075f53cd7f213aca7fbb4b69999cd1c1dc41efd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/2de61eb78acbce26e051c3c692817aa025afd02e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/9bfd14f514f4e490d37907a74f7a760851c4fe33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/9ec682e8753f4c6db0e5f9b589ca511f1580da26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/b3bb8398cd63a936506ea03a41bf07a7c8020d00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/b3f56a08c0723152ca3ac769d52ed081c6d60ecd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/019025aca03532fc8678ebd6dbcd7af0d563b1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/401009b11e4f8963ba73bf3bbfa36399ab96d32e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/6b93df3175b32d39b16b91625ff42eb916170ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/be09c7f21b8ea2caab6cbf17e80e17c879439547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/c761895d3194e6cc44dd82bd7a0a9506c8c83256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/d3f28d1f4ce384279dc09ab4788715840d0490b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/1176df86feed3d3f4721b656a01be37601890ffd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/4cd2eef72f88c636c3dfac4ab6ca55f8ca4f3403.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/5f6a60b937b388250ff4c52cb175a9ce5431b92c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/5fe653643f34115e6216a3361c6f67fbfe91cb3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/6502141c70b7cb20ceb2f9e59b937e51e36d11f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/6df7a9407fc84a40a073d0d5819f21cd418ac114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/b98e34af6de25e81ed1f4f5a7d63d4d81d06bff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/4177a7e47036202d1ea2c9a492a0fbd56c55ec24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/5e78936ab3216f554e7b543996d1e73bfa02db1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/7fd8c65fd48c4b14e4590cbcb0267c892d7884dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/bd1ca85a47f1b11266d35a00d2bbfbbfa6914d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/cb4a3a3c7eca95623bcd69350ac401e444ba1a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/fe4a32cb736ba2563489b5b7946eaeefdb3198d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/08ee0d8d211b598b97d1d72b6d91e1cc8c4eb0eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/57471881954868cb444bc53edb8166570b5e92bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/78436dc7f217c9e3fe5f4477e3c8d198f4b3ab61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/bcc6ac5e00dd843ab420a08b9756df7ad92599b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/caa1638cd9ddf98df9a976294c07777feb71eb7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/d04b841f36829713419c9a9ac80b1b1b58347d68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/2e3b0773f03a6f3027d5d25ab03b2c559b891bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/811e5b139a5cf8f573e700a5db58e92d72c9a903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/a8f79a63056f3d0d813d2ee4bb6b96e4f822e962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/cb841263e979bf33d3ac8dd158d3e079f53b1e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/edc426208a3469895fcad6a3426d8873f56b8a3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/f6cf95d858f7549e8d2242dd5922507e33551c53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/01504fe4f3d8d5cf8a67e989b6534159bb25464b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/05c6a1b2751e78a0c8b7a8ce3724b37945ad33a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/4294f46837d420deeafe6bdf0e4554f86cc21b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/980f8cb8f2e19d4e1eb1137a73a56a31c80b00b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/c77992f5c71d2e408eefff5bab133a53a2b28ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/d2b505e8c44ec09a48f3f2203d074e360cd811c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/41ac00e274b674ca6bde0f1599c49bf3e564898d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/43004891ec12c077c48604c6dca49aa132aa351f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/890062d9adf53ce38a67f24d2d457609af6191fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/a3a9da9560a912faad414be931ecbf1c16e6866a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/d9f962bb43c948ce775a5cba076962d300f46f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/dce966355e55c529805eabea7c6d7fb866672fb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/ff3c416e9b43dc41c8990bf5185416dfd25e8f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/0f53d879a8df4aa922b048be45c086083102c371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/1a3ac1fb3ea7e98fb69fbfe16f2cc6510b715a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/1a6a3358634281a515876786c256c81fbca25cbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/59c788657fad7a4cd736e3dd0510e2ca8e7b7a1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/8e739f8de3efb54484d05b125f6d2992d3306409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/b9313905af9b30bf1fdd0879d26bfc60fc26c0b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/7666351494903b05a86780920a5663252bf42948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/a939c1a4c90f1c8761fe2e4048fd9f288be4b0f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/b10866eb08cc7a3872ea6aa9b8f3c9a50348b790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/bba54c1b1c2652e7e2023f01af77987cd165b69e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/c81b0ccd08d219ed01f3000b2f2306faa7144e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/de8c3f610fd6ad3f723ca770227e0f41feb096bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/15929a193a42282f3c86f6a6460443b2211aad25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/41a6d0d71430354f9326518ab17c5c1ca05adcbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/4a3bde5709e4a4958db85ecbd4d337ba698a87a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/4c60144ff4a2ff3a7fa14babc27bfb2b3b289a7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/9f4602330047a8cb6ac6d72795154bce3ebfed22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/b25b6ce669a9a53fe9486c03646c7a3fcc3aacfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/0b16599a9705a22efdab3825e372ca8efc1ef1ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/22534dbcef6ba17c6ccdafafe8944bf3cca21396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/6de7d979fcdd39074a5ea7072a3c3d6b9abfb80d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/7373bae97d6654b434bd2712dca4a8ba9a0b5111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/b7d6f9d479a150a154fd9dc54d2f00590e45e0f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/f5d68b457509858f3b88f06a6eb2ba80c3b57de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/3044b74216cb4ef101427e9055a6e83fb266b9e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/41c454fa306c2c5c92e3592fd840888938652a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/6828d5472ef066186a14fd81487c4f2112e81b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/87c44cc3b7248ffebf6482079095d6b86798e59d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/9ad9fea4201c4778e09d62d81db5d7fd931e2528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/a98259084d782ef763b0c694641f31295f4646c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/ea5b2ae24546b3585acab9847c9683f87cedf04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/254373b469cad7d3047b724a1a03fe67cea6b0ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/b7f5583eb8f31a0295e5bdd3d1a451132c2023a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/bfa50bb238427d2ce09dc8ac69d07de0eee03268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/d56ac387fb6ceabe19ae506e3918b5f409f68c44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/e9c09bad6256f3e0aee8235209c7f848c79121a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/fee7f799c3a627ade11758530933f533280437e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/325cd103d3877fb3bf8f0f0d0c0cb52f31811f5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/43bf971fb06d7dda2d67699c9de192b6b102c900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/4e3429f4606c6c269eb5938bd09192133ea50a74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/ccb6c54da61173152897c10aaa2efc522d429473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/e0d14e89bfcc37722ceb2cbfb8abf9874804d9fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/f87a0287ecc1fb7b26e2ea41bfd1d4c4668d0d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/41e92fd4994d1084eb60007e1a626ff21ee6c69a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/60e3032ce1c2fd89534000021bd7d03102bf57be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/79cd121f9f13e6bf76110759c52d3f8de2a9d1bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/8e345fdbfefa27bf1022231e0296c2e619af9f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/91674d262086c067ccd1aaefcc54ca3fe40a5d28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/e6219277175f46aaade25d39209ab5fc1e7a2edd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/39f41d5cfbfd5a1ef509d8e876c0200601725991.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/5a850c609dce43b5916f9e63373ca908327adc73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/a351c068e6e6db0bb7e1f4549a59021482b05458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/aaec1fc8857b7ba486c731cb682e851e8b9373d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/d52ad6732787cb575258d6424d3ef812433048ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/d67838232db5255ae5b09eae5260f5a70749f2a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/0a4f2afc871c7e707678b97e3bb5c2eb4348039f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/2a13f495dee57191acd175b2c5602bd7f6b64b81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/5a937f318963c0a6b3eac041c4daf1ae81ee6396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/6f280adbd5c5f47a176abec6c6f0ed986d6efd91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/913826c4e44cbc1d1ebd0eb3eb91c2de61f33e42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/c9d4120ef8b061528b2f62e4eeb221ac3c6354e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/fc432afa5443b9000ccbf79ea99fb2055aca36ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/331b0a83c28d409ba2f9c027796338c5d9e7725a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/518abc547e9ccdf15c91a80b5dd80207c1e2d591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/706d9e0c42319053f63a6d05bc6870a9c5307edf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/85e7ea792b3a9df8796b68172ee4e5901bb04ce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/cff6907b38b71eeb50b386ce2e6a4089845a4508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/f73d8fa51f5a74dce87d781eba073b963ea8646b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/00b136b2b7dd7161dc40a9ab918d8a0c488b71b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/79828007b249031d0e36d625574ba8f8e419c1f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/90a5d7ff42c0eeaa171afe1460c9d21035b370a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/b12f4146f4237e3f44c55ee6038a05a4f5a37d1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/c277aff1ea5796806b2621a2c9bda886f7494be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/cd9783eaa6294477d6c7cc429e4e65e44cbc65e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/235561c38990414bca31ccbfbb3e866e66de28ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/76c2f6912c6b01d52afcfa6abcc6774bae7f082c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/778ee9b9fe7f180ae107457c30a9a413f35c9b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/ad21ecc96974af3e76e5970788e488e7c92ac4ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/bdf082ad7c4e34fcdfc624a1d9d7d10d8615f05d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/f2e6b5573e565565c3b07fa43e9ff1b03705d3c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/17a386a9813e65dc722b9bdcc0692cebf70ee588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/61c666a8ef4617c11116b33149a35ec6dfb381b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/8b230175584a11cba0c9402caf80d5f5ae913962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/9ee66246ee629a6971c7540490b7f878868f9683.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/d4a3b123f707201c7f130ff42f6b583c7e16388f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/d877ad1195937aa423968365d1989fb5968ad2af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/049299d38c2ca8e55aa7ffc78a67ff57623200a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/1248cd0ee681e1cba84b1667e3e4046bb013beb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/2fd3b049cd7af88ca1eaabc8887fb7ee1306660c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/5af1067cc94f74173859e141490cc032d7cb1468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/603882dbdd07290cec73f2c99465f95f693a7dfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/683f63e2a02b6d615e9d115e5ab093ec27a8e4eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/a23744217936a16198c377e32f8909c23d15fa9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/5de0b7076031e580540ba7766bdd7635a45a21a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/838e9654e60f950c368d2b21f150782a4808b41d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/8d5f5d457e74e3c72a7d3e704f1cd825d6462b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/a6ed88badfdebe4ad905fe788af88224f6201e74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/fc14a99e74ea876e2a5827fa5eb8e4135ef5682b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/fcb4d49b6af5b6656a40f06328e3f1398ec84b0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/23100db67fb0c673dcf96b05a58cac436e6c0490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/42d7a224f3408c4d3d07cc281d187bb704799db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/5abcfd95d7d46962278adfc9de679bfd039d39fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/62fc9df0c3d085d8df3bb87764a8b0829298eedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/a075f4204ac30b6646dc43589aa9eca6915b84d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/e3d8a018709f43c9e8efbbab179b4abb8a977567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/38ca9da3b14bf96528b12bbbd76e0662e94107a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/7a63b953ab677d71c1a4b0f2e95a3f586a24bb6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/88fe07c104443d9b8992e0b123295c73d651f3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/98c2c4b1a39c916e74bd8137226f0b82d7004e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/b097cc328cc96908679bfc3ded8ca7ba96e89404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/c15963992cba0f99a5ab5f300fcc8500d5fe82e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/17504a832e8c81d2a9196d6f5b5803934164348f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/29c02cb8ed1b5e8d4b88304d9cbe20b4684e53fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/86d190d88df92cb5edb8c92277906263c11469dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/b02cf00dd598424eee1c3ad0db4c53d5d1903050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/b54ea3d4fc83afab5945fe4447ecc2dbbcacb5f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/e83eda467c9f16bb509fcff36929bcd7fd49ec42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/138fb4bc1cf47ff9f8e21aaf401aecbea1dd30dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/1fab41fc6a4aba77eab5c9eced06271d12517859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/4f616c0b0ddba6212fd08a2daaa614cb51a40acc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/56c461f908d7da7b0ba8780b8ec925a2f9f4c1a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/8e8bfe7a9dc35268e47f694a8de17f939fd3b608.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/98bfec8aa844829096d9162226856604d9b46fce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/e1876156e00f4b030508f62945ba09a762f48fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/0aa7630e8caa014751321fb05005244ddfc7df32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/2ec122809cc1c8bb2abd2df523072c48bfc92c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/4eb1024c8355061b13911df3891614970533fedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/78b8c803d2db45c1750501c09ff836cae44f1eb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/cc86433d64ddac0c42cae515ffd7ecabb94494b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/dd36d570806a364645883c8c1f070e2ddfbcf93b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/e0fe4fe6905e051eeef36a56c196097a361d8a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/157dad41aba2934dbfa0eff685755021fc4c8641.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/176fe04c5091e0d9a2113f5f09a43236818b8434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/183f1c907fd85fc06f78257fb55089b4e0ca1068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/18ad4c8191390ca9f90a2fe3d2ac3adf86aebbff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1d1561222880a57610b40d4408db6ac8aeeab9a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1ecde3f46a703c4f5c70d4dfcbc67340cf543a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/261e0482668fb2eb3c99a06a432b2c22881ba6a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/26a744849642af09c2b2fc163c470fcd61600543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/26ab09bec69315d9da3e38b1199d972cb2a228a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/280a680d43f667beb6043fe647df03e6673945cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/284bd638fbab9c85dbd5fab46a9821ee52f7f933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/28fedaacc97490e361e95b495e31f904684c59f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/2a46b3022c003c12e2fc7287838f9e5d1b1645a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/2cc75c1407ca1133559877cdb96f6ef3147c3aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/3172175ba301fc87949cce5c7e2f4660204b529e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/328d7672dd7393971441c4b53731406f6db35f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/336cb6b982bec9acc6a09fbc8450b66ac6f6b273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/3a33c9c17d77131a72d37cd9235e29c94f800fd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/3e090eff07b1aef2b0741fa84b129ce1b8d56962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/41b1d4b37ff9bfd997e3939a0a98fb8699c845a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/4260ffaf843e32f751531f06ea92cf3ead5f7ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/44b788b4027426b6940f16f831e55c5e83126055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/4aa786580981f07043a1262fcb14778a95a7640a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/4ae92d0b67da1095fbaed144d9b428e4905e67f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/4c42272bc660a9ed40aece8295333887add4ff7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/508e4088bd80321adbf15c5dc3746dee70da7e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/54f06e25cc2d62ae5eeab103b9f1ae6a191568ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/58e1ea0dc2e3ea68779a87ea25431ce93b97bb58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/592f94c705592bdc7301720ec759be576b9e70a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/5fcd4f56d7575f958ee4446daf357de479bfa41e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/61711a244d37f8c3ad78119ee289f03495fa236c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6772a4bf1938253bbd6ae81787ebafe473650e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6875c9025f63aa1f31abe452ae3f4e57913160a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6b6c68fac0f72a0168356e92a0ef2bb7cc21db1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6b9dda2b89e101be0a68546ab17f8f65f64258cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6c392a22875dee4ede6e6b7c0be21682cd1e3635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6d3223485d19cf7451fe9c6ab2e29854e87198fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/730226f53cf49363b33951dc96eefc984cb80b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/73a376fab06acff5e373b23caa14c96f54a11aec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/74e9c4ea81741dab7aaf76777accaf0a26e1dfef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/77069c0c14e3ba358f3645c63c44cd3284312529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/7f3c1e849f9e6e62c4bdf2476c5ffda2c41b235a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/8024cf0dc413721f3d6e5177775ab5a1e66448cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/87884243c26e1dcc8328725de5e835b8ec3fe2f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/8c71a472a1f51ff4655bfa5ec867188853984c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/9370442b4080aeb64e8b20bda2fb35fb055e6d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/95b2a520fb6b3bf9b7485cba654e5c1fb6c3ec03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/96ce83f8e786aa4397496fe91e20254f91495d88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/97855d5f04e72b2a2b05a9330da5144d086c0214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/97fcbef3f81721cf42c66f2e3d7c6831f648caa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/999a4a2536e05e44dcc45e7bc68354780cacc89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/a0f0e9becc5abe51f8ac5f349f9975e75c1d81ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/a3d9304c8ef81304a020a2998122a292732ab31f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/a57adef05e4b8639af8fc1ceed376d9ec88ea8b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/a6ffbadde861ec55765bfc3ab83ce87ad3599131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/a9f4886cd5df58653f15deb4cd62789e0e6b2bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/ada4e17b883ff5df3bb72adc42e3fc618aaa785e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/adb98a785d33c61bbae07e250e45b665dffa24d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/ade63338b25df326ff42316d2f82cb1a5e476345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b39a7f536cad413986b51b05ffde33db65714b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b4874b1a50726d9273064919376edaefde5662ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b4e803c667d6d4d7a281d6203fa6a995e4140433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b709f1ffef0a289615f5cb1e605cfe962febc416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b71c6e4cb26d15c05e859d16e6b12a159591dce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b913b3abaaccf5eefd2f8dd67609f7b78065663e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c0f5e9b1b108f5c33048f3060907e803e5a1273e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c337cb0085040c007df244a2b35fd2d4aabcdf79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c49525dbb04ae93349d5a34cc7576f7258bb0844.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c6210b5ed6a84f4a56abd7779bd5893b2d74ce22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c6ee80110bd2299f8bd32f9aa2c89447904c8a7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c8639f8458b1e41b7f17c29f44d95d53c4141baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c8846ee9030baf26e5374348fd833777cc290e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/cb660b9da9e1b6bad61dce26e904d612b360b15f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/ccdd2a5af5332b9c1e5498fe722bd3e647bfa081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/d04c1ce12f5547070b3e65dc6a715c4274eaabdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/d0e41eda6f061571643447bb736303d430a05dd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/d4152440482ee760f7a97fd314b03524aed5eedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/daca809123a77e3cc423f2b3a2f161370fb2d12f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e18bedd7a971a2a75bf02ecacb1048f58e4f8f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e210826fe4f02fea0edc1bd2e2809391da9eaa1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e5d73eab40f77bc5ba8a628fa18ef2132eab0d66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e6168478138de8ddcc3f53f296b05795ace4c0e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e7e6bc55be0ab4f4f053a71e491aec2a030aed1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e91f55e35b0b1a034e0e4c866c6f1cd4b29c33e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/eb2e3923035bffcd4f7f55288f8dcf0f17a0893e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/eefa47b11ddd7ed6c9a85154f808190a38f22354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/efdcdc66eaefd9fc0a9c0d1e1fe63aaa78b3a335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/f3f9631a55b429bf0c9cfab15b9c7d0f068b6b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/f7b638fc2c4a1800ec30afa56d748b29410b2469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/fe3ee0f7ab2b69a648b3e7919097e4ce16f353ad.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2017-26/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2017-26/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2017-26/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2017-51.yaml b/configs/cccc/dedupe-url/CC-MAIN-2017-51.yaml new file mode 100644 index 00000000..85f1cccc --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2017-51.yaml @@ -0,0 +1,413 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/19012816226afc7ec773daf753b43f052f4cf775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/51a652026ab6fd24950eabbafa36bf22226c8d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/56df0e2c472deddbd36dd33edac8fe4c64670044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/5912e4e84c3a03fa2416361cdd27363cae58313c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/7379e80f09fc76081eddf84a79e1f5bb20893dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/a1ae26a39c74d120f187b40daf9be6748ea045c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/e56436663ce5d62799b62ef5812b164f5ce9fa70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/f9386f2455e3a2ad57354d39a6603aea89c9edfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/072aa9d9f05cc2385438d7c74ab7499179f22765.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/3ef44801ea07d66aedcc5d3d0ed64e96252fe6ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/449b5386dee48908f95ca5d995cc1ce86d653ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/4abb693760b56150e06e78788ebdd02f33f95bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/508a8a3a5e0eab77d905ab27e4345b4302a80228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/ad5dfbcb8124cae3229bff5b2fbb4c34ea2e3b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/bdafca70add3fdf995fae8cf98a1a4815217b05f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/e778ff2fbbf8d38bbbd249937220603f1b930bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/244dacb5bd596f5cface576658e7bf3e7f2710c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/4bdd3ea7db6fc2c97398eb71e079541d466b3bcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/632eb3cd621e3c53ba6df97dc657d2e42767ab71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/687e373830ed946e325942810722375c6824ebff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/9f8d2209f493857a189354c98275a09f84c696e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/a2b53a1db2448ffdd1c879a08f801b1a4cad406b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/a5412a652fca3ac0176d588730a244fee8ce5b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/cf02be2be367459b5d021edae32c3482a8422b21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/0690f86f0fe42112a841ecc7b9bd47c7ba05af56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/36878614a78f406dc4eff29c995dfb67d4826eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/3aa824eb1a1244783f824d41f1300a5c27a861ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/4a5fe5d3e10bde52744c9291a9630a27f7861f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/98174f2dc10cc62054cab7caebf036e3b621cf19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/a0201ffa46a1069972cd1064790aa53b22d65fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/cfd09066812d40645f5953e56bfda822cecf8d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/e77820d9dba6e00773b75cfacb62c1477c2427b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/033c3b4f5d555220b0581465fbca0c074996f804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/0ad23aa7a176b6a9ebba856428577fce5563d1c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/45e04221104071c3d139ac4b455eeeb43afcf525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/464b48a5b49161c80a75e1610ef50eaa7049c4e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/5ea10c0e24a5088397d0721c6f829f11f3579edb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/9c235db8fbf6dcfd01afa54d93e0dde534662a21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/d5fcc297cdd7179d1d4d1dc33303417712827306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/e1eeeaf97599f57695ca0cf60ca2e903bc98c41a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/0c2f0ba175fa405b9c53930917472cd5310e6485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/5a4b0cc27843ca87df0cb479943731a0ee63fa98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/75a3f5d7ab96e852906b88d20f651dde26dfaf44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/78e19221333ba161489d1d2c3a674790b861e064.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/79c8de4a69072d51877a9d3d69e74f0d0bab5c79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/a4ec4d54976b8ef0235eb9b9145c85e939ebf6ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/a8be809845920b6c893cf0e8f16bcacea9b8c075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/c1b6438118435a37fc80062ca02c38af8cfaa021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/14cb010105dc814c4f444a426d80ba18e2fe85c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/74be57d82131ba1f59d89f3fc5e2f6ce5ccf69d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/9580bea6a18c11473563fe1d69a410ac2501e564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/a6b7e1735c8d5eb0ddde961fd11c3a5e972d4f9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/c0308353804e885b5a9fc576a0c75aa8e838ee79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/c12ba1fb3d13c2e0c24e2e1a90f8a021f9d4c93e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/eb7fd78b1245b7a64f89f2d77430b68f7b3730ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/edc83e68f1125fe56782280a88b76f66856c31db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/46321ed8cbd8c995c95d42558cc398009fc8a3a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/6d5f9b1f895853e962e1ce51e19bf93e1fa2a0af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/84f4765709eeb13a2387a0c3a27ddbfacb25fe6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/a7403e07869e52c395a056d85ceede340c3ff232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/c69ca3a0849be8583fe627813a63f7d501a4f155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/d11dc5eaf60d4b72e1883803a6ef2b65f26ed5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/dd8176f536af6796d8d7b7d51e12852c43835768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/f46032372a582eb9ad96d263ee02b0e93d580aae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/28fead0ab4871c6820555e051ff18be7e5aaada5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/3ae4b88b0137be600b29954d2699944cbb62b9f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/6723dbb44ff95d3dfb00ca68dcec7882dd66906c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/7c6fc41e5b9cc28cfaae08c337af6f489ab0aba3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/95e27475a1d71e1adb4f3ca8efa1d63240e0a743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/9d358281b22ce524fcf3a079c4c9e67356160e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/a85abeb5efc19299b76a4f5b7f1c58a0c3e6bc37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/d9731b16dc407cc51e1494bf6c4a69a74d80af28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/02a0ad76b8cf6db702067df3a1b8f23cd9b81ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/4dce7506b92a7a108df948ffaab55655a6b588ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/5f55d801e866de18ad436750b5e15653095e5d44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/736559fe5695297d2c930af3a54ea758385f5237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/7632bed0d5d7d5d2eeaf95f81922cda9a4325379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/80c973bf1b1f0b07deb5b150509fe2b4add0b05f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/9e3ae8179e8a09d960e06b3c68a972dfaa0645f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/e07f459f588985d58213f2bafb4883b7d9a7c139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/12240540204511d35e0ad0c33d93c88fe46480fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/39d4ea6101e783769249ff14784e37125d40314b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/498cc390e93eb03ce8b068ceccb39889a2318245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/a4e91a0d1c2d8154dabcb613694f6941ebe08797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/a7b48ec810b1118ba62ccc2b376301343d1f96ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/b427ff2a98c2bb9b37b390e0c0b6b53ac5058c75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/bac71596c875685542daca025f6ab829ac17195f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/ecebcafb1f90bb999fab1b90a0053a283d6acaf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/4b91d0a49a167195021c0eb3f22b76dda7553f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/57d684bea5e7f7ec1e856622e373a90824f02d77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/727e19d14a508b0d446c160c9edcc72a78b25e18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/78d99a49620166b3f5bb0fc26e7097068c3f76d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/9b7e69e512e3b3c2596cd4d13c17acf27d061571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/ab3efba46efaefd4d68cf1e3e90c7cdee5ac6b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/c0843e14ae8f17599329384af27ab8e0de6a4fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/c0a16d4ee5c8a780902910d9f7dcc561677de4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/49f26dfa4699356652e4ad095785355022ebf171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/601abb86b8966661ea174136dfa9b226bd857083.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/71506739b06c0e99b66ad8f00b5b595eb817dc16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/93a24cbc04d0fd44ef2c1067680d4ef6878eab84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/b306dc3dbbee5bf20841598d7115502d1e6fad4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/b8ab02f315b1ad4f6dd7b0ace511ed0cb7420a2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/c1fcf6f73a85cfaf89acf7f9cecaa332ff34d1f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/f3fbfdd475d577f89fd33bf8db7eb1695f1df142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/2778f3d6e7242112151daa25eef4146765d3dcef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/5d022c129eb59af4cf0d5e57868bfe6edb19f461.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/620ffb80c85a517261bee42d235054d3d5529f7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/83877b4a7bda2a61d434b8ea7c8f426e10f9f62e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/977f665e6bb7ef22ff0ae4822d5b8fb3dc0585ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/e88428f080a8aab39689d4d2ac16ebb75498e80f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/ef9007b94548117135a02a917a1ba26c417ceceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/f4c67e727f6d9c58f8ddb95ea22308e0cc2f53a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/4d2223fa9b0892789c8e2408ae6fa4ef0e2c76a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/6a1bd3717b2299627217bfd64bd60612d47639b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/6d3fe8b180ace36bc93e1297fc552047f7fdc3f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/9c87513dd862bd257f2a4929b614d6cca40e0c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/a3f0ddec98d03b849595d705a75e410e676967ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/ba30c27f08ddd87a9bf59599751c83e51327798d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/c8b7413b372cf648c8eee842ea27c6fe8db4b3af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/ed406ab343c9355f867bcfcc8a86776161d2fbb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/0c037679e5b17a72263c2c540ee4f6bcfbf3156f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/2fea4a3b8398011d36f162af1d39589bd3610faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/409e7c1390355ba61e63bf74dddb6aad61daa534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/a28d52f184d095b480f18ed06026b4f5f2ab96fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/b986613ea1043d407c8cbd55f9c5735384623fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/c62214612929ec035d1abf7179dd9158ba27107e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/c8568b405265874d547061eca3eefa237b4ced2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/e2da57de34d3a276e1498a08223acafa645ae1da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/172178a6cbb279d2a33ed9a14fa8661453481259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/27f2044578f4618e3e6e45bf9cc9bfc14e432dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/3171c7447dcae19a6c11ade884f05d7a203b89b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/5a9c7889c52672bfdfb774526f5cd94a9569b201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/6a74e5e570319078133445cfddcf61213b2a264e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/7dd0205378699907df7d64efd84f00e1f1de95c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/88dc418c6f2fb103f3c2ad2d3e2ac5613fc4f3b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/946cc33225ae8c156daed65acfc0d478d1cb0f93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/0603cd069d0eda52bf20d01612f597c90c7ce5e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/2c6c3a6824165d95307f76c023bd5377ddd81d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/53b3aff8e3e35564e9393f787c46d6d0dda53b38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/7edfb30f351c1200df7cd46e88c9fa1e9f825ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/9115a1238a37c5dfbda328813ce71ebeb6f29b72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/94030096738b14e868e94699275fc417d6913230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/9de2a6470d473a199dd4d81fbed67a01a0815c43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/d77171e4341c147affb41855f6c141b80a2b5794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/03dbfe97b22c0e2ef32a58e2aae5ec3f1168d43c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/0e9b01487e40c59ccd124bdf9b6ce8a8a8426058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/29753cead6b14494ea88fe421305529f5a552732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/9c2c3db38c200fd85891e2e41b4cf0598afb18fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/9c6d5122df5d428a673789488671fc3c60d403bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/d8a876aa72490750c22b56bf5ba07ed544355a85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/da3a5ee4eca677e17b40c12d72d06e715a65b4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/e52715880a11c2713ddade2accdb17d13775e6e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/031235c2bf36568f767ac8ebfe1b67ce8eb86d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/22fa09cf9381e9afa15126e0537efeb132f58c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/5f9722931e858dc0281d9672e656d23834b21d21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/7cde5b640071d4600ed72d665b91714078c9fdcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/96440dee0face7e306efc778404e1b278c6fbf0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/f0b3eeea323302250d72df79e5822a39775c7f33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/f467fa45cdf24e498f785e01fb4124310281a252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/fc58cb3d5aac6ede8bf1a8e6e84d220057c2faf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/2908489c9a27ec38388c7a92469fe7cae476c823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/3c04532dc05bce33a2dbfb84f61eeac7a4a87960.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/4165c3a1c606c1d99c498199ee45dfbf9f03b208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/7cde2275fa5a8a4642c89e9592c660034b699db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/93cc6e790dfa0bafdf21cf2103463a5d4c4900e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/b3d19ea4626373fdaff0a5914455133a7c795793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/ed0df46c9fd864d89640f73d58f51cdff2c9febe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/efa4fcace9519fd366a322004cf9bcd532a83867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/1aea24d48ffd2ee4bdde5a686a189d8810af696c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/471bff1497b64ac6dd2eacff49d338179b22900c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/6905bf02b8f3474324f61c4bf62aaca53b15347d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/7cab9c43dc7e23e08af12d89f5eae634ad8418e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/9469fb43768c0c96e2a35282f4efecba7259950c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/ba28ce15a13bcec0cfd5a72af12b3cbdb28518de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/ce9759721533e4fa3832ac73c5aaf73447d86f52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/dcd1cbd8587360759cfce7a8fce882b15d30aa0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/022e6cecafd0af1b77de3123d97f02feb9e30832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/5bc2206772ef2f20fd815d3822ac39406b26d368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/68a38d3ac2f3b68fe1923de280d06a06ddf00322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/8917effc1316c221ea02560aa81867897ab60040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/9346d59132ca0574e528d40119253fb68188f576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/9ef9f08587986788e501130f4c70051a77ac292e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/d150b1949f526aab54bcbc79169197cbc06bc2d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/dcb7af36067d64510bb5de4c603db3bfc59b9e78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/21fbd3ab078c45a8fdb61d03bfacf884a1d37b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/48e02786ac3d662e35c998567974be47ce9596d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/6876780138107266e800e756d8880682b92d2fde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/7282928c954610d87c197647f662453ad52c51fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/9d103f2abfd2186f11d5ef8751c60e66ef820a87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/b93456faa8cd4f5958b258421bb3d389082eaf74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/ee93c6a6823c6a09d1cfc0ef681cdd84e4f23a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/f74e123240a52ed1f6a1382f263584f4d9913551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/1f47a0dae8953762c0d181dcb3d85a58bacda21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/2c0f99551c5cfc256f8e3b9e5f5e711a043928cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/9088ca9059191682b5f71b10232da4d865111d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/9359540cf8ea9f5a4808a631e04288c74fdd0270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/9e8a12b2b49d98dafe4ba4f6a75f85bce01187c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/a1bf9a3b593a1910d3e9b4dbb80baeec15f0da76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/de86b1c9e2702cbc366b9951fb92409c5b868767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/e0350ab672e04085d9bccb275b63b35396776da4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/2522aa54fca5cbb0215d39fb6d5e8f90fbb3c579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/378b75f2aba1cae32cf495ae69ef3d98071e6436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/41e7067c9b7d743ccf35242ab18bb4d23862f3e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/4ec20e5980a0d35dd0c3737d8b9218dd1b397a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/519871e8d40303eb99785e6046f124a0b0b13992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/57b7f466e868a3ea88e88c97f732b86a7502a2f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/ca1567185b849228113b6ab69593099b1e96d501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/d25bc42054333fc9dd4b5f7422d463d6cbf054d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/01f86581894dd41f6ec489a8fe6a3c2a50c2bfe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/0f272ae953112fbc3ac4074f026d3e63affcbcf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/1fa36a4e82a9621f130773dc86f5d74e6a76e2cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/3337af6bb5a628d4a3a4a553362c7f36c6994221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/3a05634d84fc2e14477ffda6d3289328e3557050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/48bc74a00ae3a540e50f57dc469d16eec1e2fcb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/8f366d2da711f255cf5335b805ee940930a052e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/fc479bf6e493200ebf8e4b4d8596150cb832d03a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/43ab642ed3bdc0c131e7035dbaa6feded26fd7b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/47eeb3b9f26f69b74c22f6f28422a44c7efa0224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/627b8666fe61b681ff50cc4fe7a4e86754d16c17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/7a0b9e83edb694b385b3abfd74310f82c1ed369a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/cabeacd0cc60355cd566b6c912a0d0ce255c9a86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/d71d552540b767d71126986d2f5e8d41b6467dd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/ec08c607c008a82226d5a9bf6058b259790aca9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/fcf961c9d9fe572a6e59c6d379ecfd5057163840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/4be5a32bc3414da6416e38464f4d31d2d2e7d120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/65d8f69e8ba89889e201e854265e17911408e7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/72d964fd9b395aaa4f8d0b768652be457b43037d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/9d980519ee158f1f8c40887b2c5594a0ca6eeac2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/b14a1a32ec5d8bee392be7f853d0f819213397fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/bb7a88bad710abd6408d4d6f7e473bc8e68bd2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/eb7be72aff4da1288e2cd58daf47bff37cc9c712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/fbb1742630b7ae9aaefb1d3e47f4b279fe7b0a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/07b23af100b7987ecb162b1a13ea4369cf7bacc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/07d5d144bc4a0b1c8f760ee38e7f759e5beed984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/367b9e52860e069afa11e848d0134b93ee83edda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/5dd7cdb40642f604aa0fa3fe04657d2aeb719723.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/6e0f00ffeb0210a1f62ec87768184c2668faad31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/7c9edf5f4c962c6c9294d3e12b308f88913cee44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/8ebb89ce5836d283f3bfb192499929805e23b3ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/c3ec58d8715a8a9d2e686011d3c7c4f25ee62902.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/0a34c701dfc25a816793fc0c3fe54cfd6efc249f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/17b3ce2ee41ce42576381d06493c8b65c277ae6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/a9909f9ef22eefe235bce7e3335d70e024982c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/bb5d708ba74be00ea78e66755946e5c0c5b22574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/ce022045d8cca192774b3ca049464beffc94d982.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/e1e3a66eaea51758f5b1831cdd3d56df5ef5acc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/e32a132780f01d59115a2e7ea33bdd2986f1b40e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/f7beed8b7613d1f2da793a12def36ef8399a24ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/17cc505b53d7e6cc0857a427346812ed9a2c8ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/19e60858b98037ac7467743a3ee8cd8d9164696b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/3ab04e17cbd98328b3dc3034c9c65fd9aab5b3fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/52feeead23d381d63822b76800bcd705acca422e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/657551dca4be9e3667e96d4980b14197e1b7f1c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/c98c0bb98e6c84513e8d84706e2fd0cca6103a14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/d65cff6e0d3e47f6d6a9d768c3b3e8d5bef10e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/f3575787722844a153c9521becc6cfb83cca4f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/065db35153cf51b4af4eb5d6042baffb8e177ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/3d98019f99dda390a199fa35826a73b76a48c5d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/47e8569763c2a63ef39bcb077f7dc83a5b297255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/4a7e279666af0fcf795301a357de55fbbc60f3ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/774969210a4a3adeb769b09dcb0b255852b6c8d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/8f9ee7832859460dd6f097c7151e7e0236793c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/a3f6f4887e7af880ac24cf5ba3be531bf46a4be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/e1ad58fc754c644981e56677b124332b82a90527.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/2663861a98e4d32e5c75f5a2e9fefc2a08ce6919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/7865959a932f1708b521940630951297a27f15c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/9cf43223d4ceb018b83308ba7423a0c5ea061f02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/a65f67b42940ee47d8ac19010c508c62bcdd4a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/b0d60f89a5bb086c9c0be81528bc098ef9ce45a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/d03ade8603002413028546a586e18d08e283f136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/db024dee07b018a13909958e212332397154f812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/f9984adf0808a092b31097e3ce81204783cb05e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/3028f9c66f61bfa4da4eea28af0b4237b85d24f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/58822d236a7850cde5e7555cca30bfad716bcab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/735ca40e9ee21c9c1e3e7d3d5d72525052951f06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/871888543fa315c30c77f0eaf160e87daa93dc65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/9a9252f22bf69794e06c2b2b8b0d3f48a7956bdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/a467ce802d980dee776af032713a207784fd5fa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/ba4202a22d2acf7125f1f655815c8e552a032fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/d3004219a2a2ce4bef64f9cc24b8af4026ef2ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/25cc284863d1fbf6d00710856ba36f1aaef1c10f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/5d16d7a70c9062c8aaa0ad0976b8e0ec4a2f97a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/719c29a5c8f02bef4d238cd61e41f684a43fc2bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/89c1c51dc9655d29a9ab66fc615b3b7884280c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/9aa96c3d8870459b9834d62e85e519f41653a388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/c7d902e8efd43ae35a39dc1e46afa3889d8583c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/e192a444118afcb0af6b901b9ef8f7a416a64ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/e3d40b440856f11962d195524d966bc761526677.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/4ae30b61a23c7311f93c3b7286b9f2c66c402867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/5e134b5faf41c7227cc58d3014e850df5c52bb29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/67118eca7bc714bc9d38fd227529e278ac53bee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/9cc74144b6d0627e0feb3cbf03eb573065da0b94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/c2746d2d4a597df8933af3e21b43145e97413de6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/c41a4df4a5647d8513b3dbae89d48cbfe7a0f326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/e24084caea4c3dd0b903ce1eab9b511b784ebc54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/f7a3b703606b97939eaac8f8e268ee99343211dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/10efcbf5a5dc38bbd9c8de3d1112d7d5370107b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/1ac4086b68199026a511bb1793f52a22474e8140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/1bbe5e59e02f2f8636d16da10060c6e07e3d2d79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/460d38358a3919c15ef151cc17d01ffbdb33298e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/74114460b11d429a5698f9760fa9db946e982515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/8a5e6318fe6c50b58cc6d859f7848b7047911924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/a05a3321f31812380e98f10bb5a280790b37c2c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/be50524c6377ec12736088eb9ae77c97bcb0b3d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/12d74f2be05ce99a0a0b677c42f5de019986410e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/71feace8fee7b66c05196c98e71bf5597dc20efd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/75ed3bdf12660f45e6f5704f22bdf4ae7bbad429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/7eb1b071b85485510bb558495d813d1ae6901824.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/823c611cb4853974ff51ab7a60966ffbea87f07d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/8e38b36112486d234ba52e79766af62dc05311c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/dad91e83c81533491a5da6dc61bc6c659de78b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/e2b1691cb13280fe28619e70e5a3aa8971323de5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/4af2ed600cbc861f6c5ad14f99a145ef7b4f49b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/6e0231fe4939fb39faf09a1122b7f283b1694f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/875c1e1262684be995898ef43643c75c2ba32f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/8935d0aafcd7e307af45d02fbb35402aa7606651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/8bef74b505ad441d2fb48a1183fd4247de5f77b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/b9430cca4234dd405dab34abb6b14b4fe22c2617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/bb72c9570fa3cb10033c9b8a3da4a4dedce573f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/d7607d2f88ffe8a434cd0a775d75cfc56ae9e6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/18afc2fa1fdf391ed4694c6b2636a9b78b121ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/243207a2b24589caae626ae2b9ca9473bbb485d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/30bda0678d90abf760d6d3a8672a8960b51db46f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/465ed7da97172a3ec623433792656ae6a9a36f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/4766c4859e9cbabd83b0672ed00911f2e0fd3b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/5df5e1769ed26aef5b766339ce6396ec631786b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/76f6f29b14aae33cf4d532f9162c2718f8ed0534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/e6e678e4e6512bbd98119b5b8fc670ff07883cc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/183b6bfb1bb13abc35525e6551870ee090a5eb83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/1c61ab68ec656fc834f781b6b8cc0e554557dbf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/34be67202bfb34aa168a11130a89f7eea3522356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/a49aac3b043353cc2e3c9176207415f47f227c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/c6767f173eb2722d27002d6c713495183f547ebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/d50150d011409c055cca0745f23a06244071eb0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/d94020fc15640d32dbee3a4cb3b2eac9fcc525a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/f9da0dfef15be6acedd9d799b15669dd82ae7014.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/0dcf1fd408c483c4da808b7af69ac74ae3549589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/1aa9c5cd2e93df0f054a5ec0bc283d958ad9333e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/3fd191a4477d2d1f220e28fd6f80d383f7fdfbf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/8267d7b7c592b4806e4e92cd567131031c3d3e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/a306b747ad7228f08c15567aec73ffa54a2a5215.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/a49c5e3baa928ed4b3d14182abfdd4e407117b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/a78c7eed36464bfbebb65eb81d6f00ccba430622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/de8257bc342e65fe4d9ca4188966141fe67f4287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/16cc704555c452af84199167be0764d13594ed7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/2c567975bc6c6ec1912f63a8285f66079f536377.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/53e069cbb14cc6a09fd47bfa4efb1a6a38c88739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/8104f4fc17813ea99af7845e7859077f9d1a3809.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/9b4dac0cec340ad9ab0ea1ae03ecfae58989d4a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/ab62f49988c15f032468651a225132c23ea3e5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/b6b387916a8d75294ca9ed95f4b40abe6e9534d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/dd855d34236db52009f239564a3b769207a0c654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/24d0898d9db8ea2555ac50d638ad80e9c6fc8faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/6c3b48be8c636ee35eb269a7bf07a1579875350b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/8b09806c638f3ec711bb524a7381775f98cc214b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/ba0e1837114a241512c0662aaff28e2b9d0c0e0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/db4ed40cb297bf38f69ef427376345841ec8bda3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/dfa979ab02e8fc3f3a8eedc49b27335f8ca71aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/e1ad44a58d78321232f25ab382d830ad249b8bb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/e1f9b79488696fcc3392e958d2044945961f7525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/02507bd05c27c3815bef90f6ed04dd4c1f6191d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/2dde1d62dd9ffca8c3d30e120c4aaeb8e6b950ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/42e6e57e1111fe6a8e0924f9212eb0e15b26821e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/60482f38e5d6c5bafc6d473f783ec965e5433267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/63a0a267c0287d237c8dbb820761dd9dde1b1dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/a2de9eb8e8c9dca65ef8bb775fbaf2826ef1f000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/d77b06392beacea854211c3d88a9b4d91aed8260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/f416d41123b13e82dbeb908a66e0912207f578cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/266ebff53d58fa4af85abdf89977d389d6406ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/71f35997b57d02f49f095715795ce08860a2c278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/78752e60cfc72de12153e4ef66c2bd70eaf4eb4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/7ed817447e59539c0a23c1d5436bb695c52d02ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/aaae1a5782adfd63e1481d924af8c47991702f50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/b376e0b8ab44a488588d0941782d367c2435e9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/d9722f30deadb417a870f9d6f21f7b733af80816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/f08c2641aac7bbcfda3f7f342cf009753bf77e91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/15689602555c873a30c51b8f85ce8f74e989b243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/30f567f7503ee32caf7e22a8c5571222d4fcef95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/50da96ae9cdd8fee5ab8f74d05e61260617d260a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/6ff79883c9252e18241c3254b468efbc073779e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/922e9f6ae1586688783d0d153deb7efd5bb18cf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/c66e331933280ccd6ea26b023a73afb03b4b3095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/d1302fa9c289acd736a2452c7762177962850911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/eb8b811497497beada180ff7fbf639ae32943e96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550986.47/warc/1a9f50e7c425b42d7ddbc3f9d55a2391600b458a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948551501.53/warc/2018bd5e4d33b2fa84b8eb18065ae92f50fff40a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948551501.53/warc/9c71a3ecb3f2445ef0610b5ae88be4474878e25a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948567042.50/warc/8adc04ac5e942134f59d192eef6e3d44808d903a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948568283.66/warc/6bcb7556c5fd4ecdf88a7daced086d98132cf3ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948568283.66/warc/9a50024e07e01c8c3bffe8d59b4fad5f8fed18f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948587496.62/warc/99a3e746eb8de25163ba9f3f0024ad9643c48e7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948588072.75/warc/c2b8464642e33b8113d920c9e8527103668d47c6.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2017-51/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2017-51/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2017-51/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2018-13.yaml b/configs/cccc/dedupe-url/CC-MAIN-2018-13.yaml new file mode 100644 index 00000000..ae2e77a8 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2018-13.yaml @@ -0,0 +1,821 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/0699848c4a7a55492bdb0da35685f8905a292ebd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/09956a2fa53ce7fca30545bf4deedef1bc4b04d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/74faaec471627838c0e4d3d7834cf81e0629fd56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/7565faf8ee9626b3c620bbcf1c3b3697e98b9ba1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/acfcb244e02578b46ecfa0957f9ec21592272347.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/d86e8475b1d159b294f8840d0ce7175a6fc414ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/f35b77bfa4cd18dacb263126ade1f57443304eef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/f527fa0cb0d806b6fb1118353bd292d374db4f11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/26efc8e9ba19e67571d94197934ea5f92090c695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/50e5002191862f668836cde9dec262e8709f5a0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/53cceb1a96f4c102b91abaa45fe5e6165fd49334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/99297659bf11caa34ec10dd71aaa3cd485a38163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/9f5bc9bb840501278508c075d45dd3e67c52a690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/bcf43b501c701528f23af3d22ae66d976da6957d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/fb458c14b5d00eadca2246e543aa4eea7f991a7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/fd69933e3030d7049fa98644426c69be7e9b7b0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/1e3708154a7245d70fd121c360fa5c82d6f50849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/2fb00746e65e9719cdc1c4ee3d477c7ba528c55f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/6ce19b1c8a1727ad27e4da48b59c75a17aba924e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/7d3c8311a5956bf08b4f42aaf1dd3532a7f52961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/b51e5156d1dce7306577ff1c716fd399d96410e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/d58686976c14644d7917214eebbb990029306785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/de364cc3fd7b302a881c0e94115c35674350b251.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/df164a106f3fd6dc49d43e15d6ac0acba9f924c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/4b42fb781bcf59acd9e859d6f16a7e328edc433c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/7ad195d109528de54b27aed5cc950c9e22ee0fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/84c5795ac20b60e2baf77ca7149ab60eba2654fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/a72ebe9eb27c9e3a2edbdd0f8ee56a75893984d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/a9b345f68b614a036ffede01d65df09b293c423c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/b0a92c0ed3cbf7142fc1093d2afbb953a4d30b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/ddd40db34e2a027694646fe19d14fefef17f13fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/eeda7ee055825be9b1816d71dec8bd4494b6f5f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/0fb7763d2cf8e4a0eeb0f28f81f3d224997322e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/178ff42d00f57bd6318b879522e4942e2914b360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/2e5e1fbc8b52e4b985f89dc54b4465861b015dbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/6dd2f7f4616a3131a683ec9a8a0c9ba80fc4c93e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/c18be88a4c8c8755473de70789ddd42db602ada1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/db583eade76f2dad511b680c4c81a6cf962bb86f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/e4cc8a433821ff9a3a6a1472eeb30d7afde5ae1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/e57f64db9e4cf7fe08fd5ecf638e5e34947462b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/09a60662a0e1ee47f2da8a626971ea6a2b97b0fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/21f552e85ef11c63651c85578592fcb63d76f734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/62fb826b2a28b332d3689c21994e497f65347c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/63760168ee27e68696b7401bf62e81de2e474538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/9cd9c46f4a6bedb42faf530d888e1824820dfbff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/bcf944c8c8b6e1269006c79b81f09cf8efbc3057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/c5210e9683ecdfaa967caeef32bad804abece54e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/ca05ab178303f926cc4949f99c97e0cd41cb5ca5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/0ca3819218bb7e481bd2ccbb840c42d707cc3579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/263cda93cc0149f10b65546cb134dd74eefcf103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/28732717bc8e1d2eb89adf16f1f3c37d430b6ba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/36b25e59de0cf471d1627dca6943a3a60c4a53e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/415ddfb83ff455f8d13966d08a240656954f421e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/819d49b32791a94705f11e097265ef9700071919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/846b4646af5231c48849140048f56193e4e4140a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/c538814f43be13a2cfb2b228859685f5095d0d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/1b82d3a8b2f8501a0a01e66a6d931da8607a16df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/26f76b3423162f5784665375fdac376782a5877e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/2e982eaa8eda906c0bf99bc6c31274e51d3150da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/45bd0dca90d09fb40d3087bdc8f7143f2326ac09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/45cba924863d064ed5c15eb473cb59a0c2ad4f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/87d09f62d842304bedf6c873ba1fb440977ca8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/f00fdab3af6082f407d8a5a05594ff1a31c937ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/f6161e1feef69b40952601523e7d378d0cf2a060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/210138e433a06a5d33df8d0fd1810457254b9dbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/25a38ffc1db1b8c1ff3539e77498e55619cf38fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/3e2f695f308dae2468b003dcff3d4b74f106813d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/b84d49a6d583f783573f399e5e7288cba2497c97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/e030505de42129907b23b62fc60c09a7f3eceb59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/e12be8b00ffa23c76e5398f45e2d78311050f3ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/e7157c21425b6f0009a867413db7b9bebb883073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/ea0f0b32976f3d176df17a891aac8197b692fe50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/0027fbe975754404801c3a43c2e0108462e05b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/180a874b8736a06ead31af0e94cf60868c0992dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/35c14830244fe7a30c12c11b7af5af1007d29f3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/4b4c588e35a36a9da83b8148c6aa09ccdc2cc310.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/5acf7e532b8fe1eb36be1dfac922bd6c7f1b9d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/79f378ab64166b2f5371706d6760f5f33972bebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/7c4237869dc2e36a88e0f23529f420c09c6af6f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/9783ff5cd04562a1f9ea572e5244168cdfcd92c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/16e2e67480188576216b17e1513354ba327f2627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/460601433fc67cac5770c437c98c22b2c28474e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/497967ddd749226072158dcc7993b505cf773a1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/4a65692d0432ad704542094aeccf55aa13b1cc28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/89e7c45c80ebbcb68c85f2d569a777011ac00230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/8f587722a5977a86f5a02673a95975ef99c2c756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/aaf04987616081778db98d3f38e97ccf2c3b6c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/dbce2759253db1a074ec7fa934d947d26d491cfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/2bb24e72ce993c7b9c32f1c42a44af25a1c3409f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/4136bf85c30b21b26ea5b3b15425f39db9d3a07e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/4807e21b9d1d380dd537be4959819f21c27461cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/7fa500401d9e04e1ad82e0716f959ab55dbfc7e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/8679ea8c8f0dbbca7352aa0fdc1f41bf9a95d85e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/c56ba42e7c27392f8c5ef783b7dd452d2fa03c5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/d1adadfdf327cf075b63de04c0de31b0f317daab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/eac7137d97bf8cd8265dc626634fae3b2dd605bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/0d21143c135f03bfdbba0c094e83b78d62f36fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/188166ce60d49b9226bc8aa3dd99585cee87b448.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/2195d2b442312fa3e39c0d956eaeb6d3b0c712cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/47e35f4ac164afefa4921a9b90f62b72f09df3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/6e93eebb860b9b19c1e3af05030c159c14729a3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/c6f0c6d6d9b8c50cf87febd8f4fd4f8a0c1677ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/d37fbb1dd795212512bd11f806b030c5b996b1f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/f3e0dac452404636353212c7d0758ec8246736ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/1325283029129764fbcb3497db8b4c212a7a44dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/2ef9b534362bf1ea4ad574d50197a37f97acb067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/5408b489c329656f3e12ddf4e1cc4a814dea16eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/60492a6fd62b8f8fd20b03eada20c5d9d6f89825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/67493d63976d79d8cdc4813634f477de461cf2f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/a96445fd6d49be29ce48399926347b279f0bc456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/c2e1235f2b48da014360d3dccbe359c90f312d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/da2353aaeec7e6b993396d01e278de6828bce764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/01d473bc30109f2e6fa62bee7db590cbe5fcb10b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/054dd436c4812bd4afd272cb16aec4a183a18a80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/54f7d1108c7fe0d8346ad434250e2feff353fbbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/5f9bb2f32a1ecd5bd6c795d33ddb79d7907196ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/68d1ea469e62bd2808eda725b415553d014827a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/be3fbbae40ca86e2882a202a8390302cd5d80362.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/f29cb36923c788e0589696282e8485358e775725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/fc746f42a9efb6c2afd73641c8477ad7480d0905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/07ecb5f73c57d1619eadb6617ae31a2b379c05cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/182cc6046c7c49e51c89d722d4f8ed0b1d9c49a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/1ff327429f44d74a7e4dd56fe73305425239706a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/27f9e743ce6620c5338a793afbf82dc5fefc188d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/627b58446f2ebc4ff7914baa697fee20e7f44e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/92c1b2d2ddb529f9e1318b51621538e7fb7d3718.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/ab0fc83518f9d2cd170b7363d6a3d5f017e7212e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/c66b463f45c4a660533dff3a477d40b0fd6e7ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/1991d91c231727dd6ec9b9283fc75d1a82343c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/254c1df62687d0fbec4e864aebc825e32b0424ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/32c5b82b42264cd63686cc3e94ddb4a6046a233e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/4e886ae84221f6692d4c9ad1ad86650171c23145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/6ed9f6e3b37b75b7fcffe133085ce917c8881e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/791bf0c1c1854363c61e594c6975e2b4817dbec8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/a503187b745092ec555e27b44e97279bd12600d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/bed5e9ace179c6a49e0afed6d209f95fac2d540e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/072c7928f6f4584d6ddf556b617993dc4a4f0e0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/09f8719edec79c4890619043ab1368cf87ac470e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/416a3efb67f3997658ad7ccbbc02474f8e4bcb36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/5018ac2a7da267987d5e4d7ad2fd7abeaa9b3e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/513bcf66f193c9be058316ab9fa80abcdb64a0fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/60e9ddee0e4c80886140859693816b56aec737e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/657d5392dca3919b3580e698441323437ae1087c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/84a7714aca7152ab1f92c56f32d0ea3bbd2bd185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/01b9aa44b3ac4285e2f407df812bc8bc0e992673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/1397958f64939606f1ef8f6bf5da00571166a231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/150eab2a95f52a1d1c089baa0d6aa8230de7c789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/18a37c60072fdbab9fc8881146f70c0627f802d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/1d1c6c75641df5ae34af73f571a34e9cb1bbab58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/61c757eddc6cc72e00215461287d9e2d36f5ddf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/c5087e7328545486d3883b851e658901029cfad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/dcbba8035d9546c6f17b685e9de713e3e56ca860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/0edd04523d8d439544a9d1ea92f7b3585dad7c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/347342863799908550a7afd41761047a8208df0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/3f5562b843dcb3022f790e66bac029dcc2d9363d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/4b4cef8637091318223eba8208849ff7c522b651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/c0aeab8ecf119fc767e2b34d4d4a2e1027d2158b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/d75219e2dbaa9d2f16cb7d64591f408dbbd0c2a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/da15659dcd06d6307981b6a4fa963425a70b53bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/ed68ed58b9029f0115603a9d6f94e4dfd5d5f260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/0c5e87c753b5d840cdcbbf4507817e3c60dbc0ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/25221316a55e9c3198e5435d792d342f27fcbd4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/53d7d7c7732067113334e8c178c8e00683288fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/827ad910dd16082504f4e6f1798cd33ca3b71d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/87fbcce2cf990cd49049b76474ce702ca6079677.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/8c93627a0095befbeca2e4440ce3a4753b8ab3fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/cda69859648df0c7c42ceb634c54111956e5ec88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/f95f087ad2e7a8fbd6e3f5ca9436dbd266ccdcad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/29c879aa6215d2323f3ee31a437531d9ce0af357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/2afac65a9d8a97b64db96f59c9912680f3f60616.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/4cd850f8646205614cd98493fd785eed30988266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/4e781a96bfdcd521da8c7f33f7a741d4679741a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/5238bc7576c79bf10dff39772ab294ff6aa34473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/7aa2bd3393902cc56b59bda41e4e35ed586e8be3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/b319f2782dca559b99b049233b2372ea989d2024.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/dd09725416c40f21ac5ea96d1c8532df3b05e507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/24d0b8f9eeb08f87d138814939f17caaa38366b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/27e97c372a54ebd246d3988026bbc9e3d80c5ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/28458ad28ef9207a9f0c23e942b945695aff6d59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/7bd6e775002fa8800a17939e1084308c82fc4af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/839161e29c404373505ed789de956283565d0151.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/99751d9211b674599cc54499a73c5125393f5760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/db9fe628e0fa8a57e3fa3ba3138694d220ac5ac9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/e665531c6bfd8be0d77a8f7d313a0cfb2abc69d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/0115d6c2d1cca8dcd078b2f62f339315d1405a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/aa8c7997af552f2d6ccb77288d6f5f3ef429f44a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/ae8465bad447e8a5cd52099208f5ff223c8705c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/bb2121c9dc6fbfbbb394e410af78c3c7365e6b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/c5a5fcab3e2a04c35d165966300063adc522c0d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/de1de157d7af3cc10ab0ebf46c97e176f81b5405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/f74a5dd9a9c60464befadac1cd9475ddfdbe1c22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/fdf9674067547699e2380351d8c707f486699cd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/1938728e6a6fcf3b97a107743fd47abe5b5bf03d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/252c9806e54d1533311f00999760729054b21a79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/4912bb402e41f8536d33e9098e759fce74e21e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/5df30697fb3e7bcb8c38a9941ab3f44e60345b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/bd423e22b0efa4acbbe9ce5120c71167cce8f051.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/d8873e0057b9e39f512c2882074382fa9712d647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/da924de22a3e063af098f629a7894eb50ba62d24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/f4a9a26b1324f77232e823a7dae830c56ee3a4ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/3cab02edf2d5769b59d0a0957414795754fb2881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/74545cf3d39f2d4a3ebebcc8bf4c7abefcd3c684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/80f9e392e27b7dc0c88f04d09832cf32ffb822a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/cadcb603664b6b2e3e251a43931544dfb47217bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/e406d4e03f196ef4ef0824dead2ea5927c811046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/e8a1afcd586c6c59e43dba8793e38719a8536da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/f42e47107275a761d0e3f6122fa5305fe5c73519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/fc126e7989d3006cb625b871d0e5cfe9346a0155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/25f6e64bba81ed11476bcbe88a1c3db21b1daae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/39598d12ab0f3019c07bc21f4c201c4cea09c1f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/586194c5a1ecb48e4000a26d1ad1d7e4244afb70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/60853caa892a4f439a78fd4c1192ab1d55d71542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/798cf7586f2f4a5b3b3d1024e5066ef49b86128d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/85918619ea7fe6232e8decb731635647c453701a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/9cfc0c1ad11d0f8e0e17354bb4a22e9ebb70742f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/ebe5e4dece1f77a6ec725d2aabb607619533686d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/2b5b528ce237a74b33e49a562fe9c286832f9b5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/320b5dd8923d2cd700ae6f8e923f8c2733d01a8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/36a8a3d9bb8e65ee6681ab7de173c0beba305e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/3f856f367e7f6aba29da65b74d5c6c608b3fe9a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/4de364aad6f6ef6b2156a9a934f9371d5359bd59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/99ee5977e9e9c1750ed7701b0a39cac7cedace73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/daaab94174e93d7974e35dc716c50c67e03d755d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/fe3ca8a700a1a1bdc97f40c8f13c4ea5bea86199.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/4ba8b69ba08202b76af17931e882c4f55567fbcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/5fb2bdbd624909a50e376b095c4798014f867b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/884a358af41a05287b58ed087e5beba9bc5dd166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/944d3aabd7a26b203d2bd13a1bb95d4f959f7bba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/94f1c056d0c1b76957b358c5d99f7fe6a772e76d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/a2d5205564783054028045ffa73d4d6bfa084e2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/dcab204accef2507953d99d684cd30c91c91030a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/e03b24a8b16a9fc1f7e0834b4818d7c71a5842e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/048d4ddd84c6d7cd90f8d4fdbbedf9b99f91fca6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/2d7aab2f4f004351716f55ded9f009d67a3ae017.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/3bca61f46bd3fa5a581cabdbccb3e516ce49cb8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/613636a0c8444697a1f119e5d2411136365b0d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/880eced9ec9c898379486e009d4007d351e66aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/9b03ad0146597711e3e27621cb4444f923656f4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/9d82ee9e25dd6b73206cdc354f02cc78b170862c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/d65127b709a8c89611beaa9ae7511f8b688e2b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/0c89665bca6087ec7033d101f22a8d635176a85f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/19b7a9fe67b112db7a5c16b1fd6743c774a992fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/2e6857058f0bf6301c266ee156daf35e69ce6538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/8aa2a77cedeaf69f16c7bc2f98d8bbcc3bfb90d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/99e7072bf53fc0495bafcd9582496966b947ad8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/a094f9c7d184af1478be0aad5c859a6ef4ee8752.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/b6ba06b0ed24c2ef8666abd16f5ebb6049fb36f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/f63ce5c431d15337f85588d585f8f1f2a5bfa48e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/1acde186b571a9409454f47a9cd4c6f5f4ca775f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/5b81c3af30d1ce9bcf4f22ee21434748aacc9c14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/61998aa8bf5dcdf3d1fb695cc1320f020787c8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/acfc5e9e60a90ca18939fbb2d2ff3663f3000e58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/bb2b81a85560c0c69ef8b93490a4f8af5650e404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/c30f82637d8cab7d766cf3293ec3e68ab3834825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/c824bbde7e9dfd99479e3d862b707d39717e64b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/e0e5a155bfb20d08f44d8a39f6ca1dfcbd982855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/360f3418ea16a8c019f28566115bf19d86d8a404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/5cba37f0e44d7dc43d0c9ce28105bcc0b4b5aa0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/6ef2ed214d6bd5034ce6c115dca409d86eccadda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/aa447f256bd8eff6b5a18c6c235c1ac7c01f9cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/c9e228491f1355ac25637697d3d936b73121ef4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/d3e27fc823c6a936a6f459d1ad76c9539dba4005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/d6ad90c047316327ed18e55469e9f164da131531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/f79807a268c1bc2f40b742689e0ea77bf5c678d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/0b7b8200b1aa1322079d951fa0c3b0897406009b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/3047666c512bfd246d5fcc77213a994cf3f5ec93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/30dac4567abb2215a213203319d7fb6bb878a4f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/37701a40da87d243aaf1d8c6c76944e854309cf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/7c5efb9dea3b68271d7b785771f44232549fdfe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/e236ccdc92c5775114f3f3b5d2217ee164b97a1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/e696b71c0b2e3be3442376c692156b472cc59dc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/eb45b561b8b866abad5790ad8b784275b76a60c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/2f4cf803b0dc7696f66b2769804b19e8feb539b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/4ed71f9a6181a188c710e526b9ae1947e1e6ef16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/4f1915617914185f118431609dce494deb75a6e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/6ab2c11f40b0155594a202d83e3a7a586a2c1c51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/710882a471623eb84b41bee97ee7289962b6c8c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/b01a34df2d2089aa1f55351dd82b3c494dad3022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/d375531df08cf1e0a81df7813b9b44a4ca937267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/dea2950cca7dda4a1bf123fca654ea01f8dd6b6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/1d63cccdc69fe157f7af6254b00b5f6463ea80f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/238cae637e102d6aa7918a0b45f36f0daa71cab5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/26929a67b96a80635eb8009cd5e6e73070638f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/28bd8deafbdff184bf6e82fb775e68399332fb04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/7eecb150c6439099873099ba2dd642b063f97781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/88b3206891e84fd787c18572ba2e808026875a5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/e2f938db0a758c8b86730bb95108704d48480d50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/f26dcf3c241fafdc5af76e2af72741ecf5473f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/01c94b00d198ae42b5b7c5bdd35dab5ef3d0e222.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/7bfb23617d2626ad1b864779eecb56c155baa021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/826df93a733755de903c67eb0a6ba46e0b4197a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/9b50e406efa23e39453368168fd07b75b899616d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/9eb8a90c11ac58b4f792a804c351a24620bb601c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/ab179e504e616a7ed6e73baba597ca1031903376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/ae27e724f8b2e5d66acca417829b22951280fd0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/c754769acbd0a2ea99c9e07d17039a3838391e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/0576a6ceb47c5114a7ad8c34e628f768160239fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/0c4f8b1e1ca85ada5f738fb12b110e037b6094a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/0e7511738168d3a8970adbf8897b79654eac3675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/417eb8d59ad67a05c4aa9dfab8183df2a6403227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/6455e0895417eb124a34d9a17f008aeeb114e6e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/7bc9083403e88fc8290ac1b60aea17ad41c1f8a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/ba784446d56479c7d461763c5f2d7f012b5f283e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/f69f64aa3a1ec0d94ff1250437004234690e0e4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/2924932779c6a2f041f1503a0669706108cbfbd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/5f6699e093bb2b5764086acc781e7756e8b3f979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/9d3331af406257a416fcacf8d38211b3ffdc6e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/c26648e54b6a0c3eed64d1eb7c98ed8d9f0bbc31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/c27f1323ed13287af01afffaaba1682655c9b8ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/c66e120e5b1f39743a9c2f2a4573198e7fa00378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/d8cab4bf6773894d03210955aa8a69f2ccb03b13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/ee4574c1de6efc25b083d9ca059920880c73eb9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/3514725a7f9add2bbb671ce1b60bd53694ad20d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/49e23bef89562636e54c89904580e035360cab39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/4d2b506823339e843d65c23f7a93a4ef242da502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/914caf6463bf15a75753fe6f75614e3bcafeaded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/b3fba441edbb166c39eb2bcd5abd15262a0fb12d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/d1c5d7ea3649fd89bbb8093b402d3d317a892e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/da9bd219057e8d3bfca7c293af22678c19201254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/db2b89c5d1570606da56fdcf0adb81af2f147b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/09c53d640c08fa5d538fb7975165adfa3d83d0f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/1f133fb354ba958640dd9992ed1d658e068e8bcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/255000638016cbfc3887b37eead047789c07fc4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/42ac7c41ef29a593e2ede9cffff9d05bc17196af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/4b631389d29bbd0bc1d359c0c35c234d08818c1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/6523f6e0f8b5258688122edb3927a5c14411a19e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/987a7e48b8468a89eef0e95f6ddefbe2f445e5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/eed995415c86078e425217d261aec8b56898c7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/0d0bf1aa9017fb9bbab969539b8435ac6ac91b43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/1aa6adde0eb3df9b2571984680a3537c4d00ed93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/231a1c76f7d2ed8d53603c702d5a39b457012735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/48d81f5dcc2827baa3de1051c989c6e3e15a0dce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/5ff1c76db345acb07f9fb51b96e9d3835f60d7d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/8257c979d1af2ff4c6811cad66d82338d56d84ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/9899bdbb64d08fe1f5d3883ddcc5ea4b741b81d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/b2dadd93a0dc694d08770db8963547bb93ae5d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/1ecb12426db89f8e769f311463d339100e0bf134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/3900bc925d2ae5ef48320d8b410ec21418098322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/791a605404a79a505c74852fc4c022a79e575bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/8467b9180e0a1bef70eb2e486c074346f5cb6e08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/959077e3e3749ecc88e654a4295b16ad467fa046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/9a42383478f99c192f3ac0116ce6e640bdffc1bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/e394786cf94b7cfc3964368d444dcb1ce2f6d96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/fa2b389dd5866b6e288647472138aebc6154e2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/23c2b23ad1322a9bbbe92e6ff27314fcd87af013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/4aed9b3a519836700d50395556a3cc730670b214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/51e5c01c8997ea30d01fba81fc3af098ea359412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/5739ee9a3c8006c062c3de14352a6660a307c53c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/5ef1866c01e0066b1acf5a1ac0a3698abb3ec383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/69580315a495fcf51e83e0c32754904008c4f8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/da63bd893aa5d340f712059883d048687a220c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/e200c70491836dd755639dfd6b5653ffa637fa41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/02419d29ae71c1f34ed86a46ab16340208c09514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/1b4fffd580f316e727424feba043e87973111157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/560bfddac29f643458443c5de6551d2591ac8f2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/676339dd291be780cfd0e2190b8e445d93f46b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/7b6127fbcc64ca0f01bc61259bfcc5d9b0e59da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/b6144a71a2973929e78f91827e376373351c3709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/e6b4841afba0d4e1decb0084be5d4a48ccfb8726.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/f624d86a62b84fcbb9703e06e962ce4c6bdc3bfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/547f282ec3814acfe194189cdc8cad4dc39df209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/71a82dbd2fbb070546c4639b4916e49365341622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/754fd63b3b937ec8056ca5152efee293c4324d22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/7d9191b7803befa8db4140706473c38a510ac449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/8afbec9ad46f55e5faad810417707b091562f8e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/af43dcf45f1c74b7aff774c7bc09b618bfd5f69c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/b9a1ab07a3dd35bde7e1bbb288170211c4da437e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/c94268fa5f6ac0ad2b56402f1b84885007c7d1d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/37f8079e9560b8c72dad26ade21dabad8eb4c971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/46cfdd869969454434aafe72d3d3ea99acb0f567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/76f03dba9934796ce0b762d97bd6b2230da7b0d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/87aab1fac5fb250f0ec3b7a3d6bcf4d9df763e89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/bb6840df7201abeddb6e49bd303210fe4958db28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/c64944dce0b94f1630725e0121487d9bc1f47d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/feb410245327b3e628776de7daa71a986c1dc494.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/fee52595e686ba5bbe3a633518483743e8c7e6ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/29852762f5778f51b8353dacd871f239c9d9dee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/3d6f55798e24b3469d4db449bea0b0193dc4bfa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/9ea64da1ab537b5967082bff047d7ef76f11cf90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/a5b79a7d57f8be69c325f335bf89ec3ac72dc5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/b5c3ac347d285bbef2b0ab4794d180609b6ed9d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/c38140c6f47f1ad361f49b2511446550fd3a09d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/d1d427789574cbb9ca731fa84729f7a8a78650c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/ecf01297feaa15ba189a8c3b0e4ee15ee5fee31d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/0fe4c3eeecf3a64828ff57a43954d00e0696e70a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/1281e9b754dea0fcf35fc4f73ccab6f740671ebd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/13ecb7cdf630733e64810d0b9dd90f8038329754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/14228e02719c6062cea9b6bcaf06bf673be2b80a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/29887ef2141e227f59f7f3c682d37acd7f85bad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/33c86200f842f78df5481602949cbff9dd1f0e1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/6ff0c1fa143f80bf4ca56de645064fb54463f840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/e40c17566308f03aecee8adf865bf0fe7f16e096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/29be6f943cf0781cf1c7266cdcf36053a469b1d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/4a3083211fc99facfc1806f4d4cfbd1a8ef5b80c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/579541b90d0e33674ace96b0f0eade8987d7545c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/7f0cd0ab582c348076827f16602b69690e2a77eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/9e2935ad7fd6252219018473085054e121ea320d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/b54ddb3d38cc6abaf60806bbb5697240802b5840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/c8a54c5eccf6fce3996a0ae2e9ad2f14a96fc4ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/f3e3df5cb18f0ced637a7ba599de01da5cead08d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/033c5f285b569df97726b77afd0b895a2da54ca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/205690a749ed39e748bb6aeb172cf785300d7c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/59afa2b7dcd6aaceb95a0670df0f089d706459ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/5a2a74d0c9d294776665e1b0b890948dc9c3f665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/6a68a06d72aa34da43501f0e4407c6108bd9beb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/a022a7e2b807868c70bfdb15d91b86ecb769417e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/ad5551e7d134317e95fb0cfb91bb6b3b00f2fa61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/e1e3e072737d1bc1bcc9e53196482fdc97ede34d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/58fc2fac2a36fd59706982aa951fc733d957dc9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/5e8a3e16ea634ddc2439b99211f7705824709891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/6939d4d0b4c719d07f4951ac262f5081b52d47ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/7273f6900d31abd496ce08cdc012112426589cbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/a71b62bc6f84eab96a7b8037bca6ba16bdcc6528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/b5ca6a6a703afc4870ad890702facc970bc4efb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/c3cff906792cf30e66fb88158597fba58d6632e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/eda0c9effde0a0dd4b08759a228a80b10c0ce565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/2d37f17b7947367d09e2345c9b94907b9c77d52d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/30b2f12bd39e99b642780910559231c5fca2136a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/6ec3162db23e9d2ba955f3332abb84765c4b8d39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/78ee771ca953a778f5f11a4285af3f9a860471ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/ad9e13cd31ee8950ff83c697d8162c6dcf616c4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/de4db09982e96585b9ae6e9791626155cfe3c1fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/ed0100be73a5d4c05a57eb680401be68020e793a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/ff47eb0909fbfd8471790fb745c3b03e32d8b5ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/3626ded89769e8887bb3e2aba3e32513b202f185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/62b7dd9438e77991cde9fb24574affb976092d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/66361e646488c9ab9586e0967e791df8e806442d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/7a1860d3f2d9a519caff7e254edde549a52d3f1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/8366f53f104dbd63992883d9b4a47d2d6c9e5e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/a360262ef33cd0fbc5cb0373a276cff111250487.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/e6d26b64cffd2f342bfd8daf380e7799e615a29d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/fd8596a6ef5ee545c7bda0ec9983e6dbd3bcbcef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/2b15631bf942ea3d6c245f247efbc73c2512b602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/34fb54fc958b47293907b2cc643d1c89f38fcce1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/64353cdf464ff5d096187ea9b4a89bcc3acec5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/6ace3864b036eb09c4795940cbc175e650aca571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/70e061cde1227f5a7f476d399bf8ef4489d037bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/ad1a578b970664ede87e2b3e5237de652a87a493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/b1f8ab8d9aa36b157bf34ea45e743a82e36f2a71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/dc94ef83837e0fc3eafd2ad3a38e26e96e299c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/0190322ed03b961a597ccf0a978d24a1b995d04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/449c9d4b6c5838c55bea285802ed384e670fc0e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/4783d2c85c9cbda710b3bcabacbb4bff671426bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/5d01cd2ece835ed6e2a5723b4861e16ed1fc46eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/5d377d4704804840a9e74020846f2f3193ac2cf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/9f90cd1e85c65363a0f2349e2b72513780f8e759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/be580afa3e010aaded5242004fc23f808eca83d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/fc7f51973bb5de565c64e7ba13e17a1af92d9da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/3706d0d24130006a2eb72ce076862894c402f73b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/66f424a81eab267b36f756330d6d58db1c244209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/7a0946ff5260bb3a87ef565cfdcd1916ce026570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/80327557bcaad765a1ea858bd1de9dfdab301c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/978e1496848a186cba502d67e99c5edbc81169bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/c0f6a0a9d65f3d5319066b363cfcbcea812ffd09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/c4f692d17e6d088adf07c4d9914e181cba11a062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/f8d01116a286376be99e91e60ca110392e351274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/150a82ca14ba74e4556073ae87569c0e657a69b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/56ef5e31a1d1aa98e624b0d808bbc561fa35a211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/70a7e6fb89d116cc387bf3dc29908a5214bfaf54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/90406c3625a811c1bffd707b2d5dacb4684072ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/9ab7639c9126451671d0008f928e1521b45d67db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/abbe56fb76b8752874621c6c93d46284a3aa2bb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/dad4b43cb0e72a6cb426b1ed804f87648daa0225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/f3377295d4b0238d4a0af21fc0f5a1cfee87ab3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/2ab94110af4564fddd0910649d7bfc374e7d7ab5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/3ae0361191f59429e840b1442b5112c44941284f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/5ca2757abe7ace830a060f8373b8a254a69d809f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/7944f8294977e5d89645eefe0c304dbc05877893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/89fbf75465d686a044daffb09dc3a7108fad0285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/ae5e7089269e5e15e54d4aff7d14317732823bf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/b879b6b6bc144e176a8155b1e1628b9e04928e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/fe10ffa5fb753f8d55b87a7ee4065483e5c6288a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/0e60f2f424225e397ae56aaa83cc07632153537c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/1f4891fbad956c4c44d9029a85bdb5591974e947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/618e9963062f361ced6d0915fedea35a9a4304c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/73e06815ab61eb1908d62b2c4f2b19d7aa7fe578.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/903d7a3160a4e1b295a125e45d37b3c7e4cb3828.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/cfb0379d99142b7eee9d2d0c6540ac5d1f91cd87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/e43442907e113c31a92b5973f57534d805caef2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/e91b50ffec8229ad1e1a6c689dfe7832a9903f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/10647bf6c8bbb27d1513c511e311da0834d84f97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/1f9a27a22ec08db39f528a825274932f064b89bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/24eb6d5edd2eea565042c8b58f765955c7515d28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/33c4744ece3be6d06ff2d73c3a64198bc3c2157b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/39c32f9575e1e40d764095afc3b17e1bf8a1ac95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/91023bf3a715f73a8b8c885507f03efb1235e632.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/b87a6ab649d8eab76e7d97ec4d067b1fa92c551f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/d3ff47697cc3eb4f626f5ace158c71b90dd3d7d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/1af01c820dcb1781d840e7a0847a725d72add09f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/25685a88f70369ff3cffc6e5557a263e9b721c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/2bc71af71ca16b7b0c52db6fd5c09268187fb29d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/b1815fe535450d6be0996f7e4c22504e158aeb06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/c476b7ff68f07bacf43ccd4a3d0c32c2d71f8f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/edfdd5bca3dc7c17a171b93fc929a28acb651cda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/f60c810d7fcfa3d1350dd1a11954a42a9c4323ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/fd8be378d3aa36677fe752aa8863a7a47d25c46d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/322cb666ff0cf0e34d1f0ea9f22b7de74bb6c6a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/4a4eae43f3f98e85a1a75cc78fdc1bbaa862bffd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/6e1ca71c099624fa822a6440c041ad35d5deb5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/774cad17893c159f65945b89e4cb84039e133911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/7c0519ab37a03f8cfc413d7ad824f737106ab44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/7de11b370d96f9173aceb569f7f75e866df7ecba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/7f081343175f953b4c05c4b7168c09d49a51985b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/e161483dec99ae77772033a546e4a2ac6fa84f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/001b76a95b1e4e6e84161a434f91624d7f5736c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/09fe3ff70c645b3e7f38adb6b9788360b3e1fb7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/212e0016e6a5e445c53c07283e0596d2e254159b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/5339a09a9e222e7011681a24bd94b199536c1534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/606aa44ff8e6b264e2d00c798a8a3f00e0562048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/73251dce0535a7ab0e2230e8068c342c28f62771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/a5c5e4e57392d22b8df8d23262e742c00b20c31d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/cac14e2a0a06565e200756be3a99827610945381.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/18df446cfe373dc92787218ccf5fe57efa9ab539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/2a5c3bbbf5b3bd955db62533b87f404e3fa4ea3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/4d2bb8832634546231065c529e85bd634239a92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/5e42dc5059c08b3216f3e18c5baf2c044ac2530b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/c5845eed0cc06e95de70465e87cc92419dc9365a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/cb5b97e8d09b60b37a2e2036155cf3f0de3c8df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/e309e8299f0127c2add2484737fe36ed7c7fdb61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/f877a4ddc2d8e9a5f6fa9bba4c0acaabc6fd4991.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/1bd3865c88f03dbb3012fd9e03eada6dd5cf3307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/729f1eb0c2a6a216c6262ad60cf096142003c729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/7e4b4cbd89359f0d8ff82bbacd29cd1bb7477b31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/7f8e924eb03c00ea184474862d44e847d2b708b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/8198e9481aede76e2abcae407c1b345840a85bb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/9d20c3a532f59f212790d895e5495c0134331c73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/dd1213d6c12f7bde210a5a51c9378ce3e0c0027a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/ec687051f2461e99fb71e02dfcc5c308ca467f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/105861fbae862e8526759d8d744a478c5d88e916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/19dae3505646fb1d5fee2f53cb79614e700e8646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/59cb8cf44c40c9a389e0f8ae0cd27697960fe8ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/6bb0bc4f522036e59880027b8f5e68384075e9c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/8de2b181c72fea87f6c79dc0e2c29c0ecfc49519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/b48130f68d1355b81e8be562305010e213506cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/c016c72066cf611f5e846cff980e4d06a0335fd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/f89434e4dca76388ea977bcc69147c69fb5fb5b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/040aa8bd541514eae5bf3f663e0a93d7af858554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/3f147b8f1fa6a09b93b99e7e0fa2d411f2ee05fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/58ccbee75b01624773c12e83b8f41732d4324bbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/61335b7d53451d208da93d51cd88bb7f33c6436d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/ab4ab0c098982c77b20d8956e9f1794b0e7ce61a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/b2df8c7d6af8aa60c8bf2c860ae40d2386a21ea7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/b5c29041d6c7ff1b641a19cd12e98b4b744afacf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/c0fe1933e1742d4ccd175e97e5524dcf0b738338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/125cd6d7d6b8ede7c438fb62717d53af712bf465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/2fa2bc16ca16c62e4b736c00638c2e7cb394b8a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/3e45fbc139756a55adc30e312f271a26fd9a8043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/46aabce28b5b7934a0af19ed66bd879592a8d8f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/6f15c87de3e2d6b07a1470fc5af086eaea3a6411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/8f07a782f65188b379d5eadb67b98053ef1f9fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/b493061b0da02499461050a60caa89815cf44292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/ee886fb31ec13cf4dfb9870de650e252d504fe7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/3ec61b85b8f5e5388d4d0c20e65aea58576865d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/4c7a7cbb5d0248a2876dff15fe7de2a887137aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/5f8283d257fbf7efc796686e3f81be2923d6993a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/68493af833f1354da22e7f3f09df90cf844f1ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/7b1cbabe1c75e6f8af5ba3487d9e62db9fa6cb55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/8f240ca6cb2f9de6e00523fa5d07598861f96c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/d0f9a4691ed3507bebe1e76a4481b27577dce074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/f9234dc45d6a405704c2854f19ca8edf3f0a60ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/252856ffcc9d9b6a5617a2799ee886028cf94a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/381e7976d3741b573df2f5477c72258fa7f89ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/4fdac753d2344088f4cce4fddd4d5300cdf78246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/694de5c7c64703dade99c8439ab7948268937539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/6fb39309cfd8031e1ddaf188312efcc3aff7177a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/aa7e2304545055fa5db87a4cd1b661e34db8b782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/cec557629f2d664f70aca611b736e927743974bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/fe184e18b8fa5aa81ad90b95199ff955dafac383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/02d5710a27712d28cd5125d41f2fb60e74d166c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/1703a14014e887356535ffb4b80ccaa6fa6df47c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/25d076b8c4f9b8857cabe1be89357351828356a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/2fd8693872a415f8468752db097a856f3c072ed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/337b40dfc59adc1b7fb40bf84cf17d5448ae0be1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/8640664d2f05847de9949b401dca79e9d6f9e3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/d576c75119fffd399ad8bbba3dfefa7bc695c34d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/e880178d75fc0db12edec92f572463b1e58e970d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/0646d50e4fd0f635011a7d2afa143dbcab9b3f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/2b36a1ce5240987079bc8bbb1a48c724e39dd39f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/2e067cc2f809f1c70ccd1d301c7087ecb70cc56c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/4ecb07d172eebda5defb065b2d062e2525ff04c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/a98db8e10aee638189db40494143f011b2c63c80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/a9a7aef87ad7a7870b65fb58acbf4ee383646ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/bf6bb27af9c510669293621ac185c94e0a01720d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/fa72dc4b6109814b4bc66e327355496be079b4f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/19b7dd0ba40025d0e2b0d84e5cb338273acb7ec4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/236f6a41ef9b664f81d2c9c2d308cbab6df304b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/272db282876abda8e69f8ca80de87330fd253d0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/40d4e416f1795cee9ea0501aa351910dbf8de013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/76e42bba800fc700707cb8ee41b6bdf905bf6e2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/a8862374a95c1dc9dab95fb072568ddc7ee7521a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/c781618d4c6633422633ad0b46ed5e5aeab17e31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/f3592fc0c6a7c27061c1e43d5baa48ea1ec72c98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/800b1ee6406d5888d37abeb96b2d966151449829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/85365c74ca331d2c8daeca46a963b1ce7814a194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/8750a4d7dc8d4b836102f6c0d2c92ceb9e77fd70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/a523e1161799c297e15317d9198be3e9396f734f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/afbdc4b58f4db89deb3e55aa3397188987d26c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/d9a76d515bbd22f3d130d0cf358bc790b1e7a483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/e4b18beb30b7842a01c5aafa86d37d7b84b8dac1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/f0f2c1ec211f673ba8c33fe2f39947dcceb6f934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/38160a5ad170384e538ea018048b8f700f1a4f75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/604f154cd873d209d7211344c1d0854556d8647d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/665af0c9d0732421965658255f1feb98efac7156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/73885f03697dcf95fdfe703a8909587ce76c13b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/a3adff8990fbd1e7bf25ed5e63e8f9939fbb0f42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/c6d899188e8fcf4a2eb4cfd1f9bd1424b18549e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/e34dde8ae88e2dea9bb980d74945a729d1b39793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/e6c0955258254b93cc79a37165e47e86f234d3a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/033b2db086bdc6c4a6bb48c98e18ca5fdece30bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/12d6e7e35c80b2fc5c105035f9a07cc5fd5f2bd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/16d79b09fc33e29dca47f8a0984cab703df476e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/75945fca454f35685dfffa5518c3df451099aacb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/b05d7453600d03f29a02e0b51506fae821aa5bb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/b683c7240496252f429b3cf53aa93ba2948a26c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/e1c1af57bdd9875bf3a90da30e51efea19c50323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/ed9e90c0680efbd71f126f7b1124802bd76586d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/47ec2a6aab76c4749db97c179baa09a4a179b78d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/511a1477e522e83868a72092a12a3a36635cc664.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/7d73a6201f8cbac53ee652b4fffa70c28c134430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/8b4b692f358eba32e4baee1e7cacea2cb79914fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/aa34a797b0b437acd1fd22bd4417bbd264b1626e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/c8c26069616beb697aab540ba0e247636f449fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/c9f6d25e447fc9db09fa3e404f0db8186402f916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/fff8aa54a18ac3139de1c6cbaeaeb569a7daea70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/1c96a555d666a57fb12d9a84eec4750b33b9d98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/35839f2ac213a2600a6a7865828dee79fe27c755.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/3f5455ec1375c4feafcbe39e358fb07c2580fca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/439bf6bd9d719947edc730748cfe181679f8facf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/51e6d66eb2f499b7087b76828d1c946a19a24d77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/9d0010081fb05aa7d08676ff86b5ac06227c79b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/b95c218d25dc0d820cde8ec4764f36718472cf03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/d998b6585607fc74a77ad66d4ccb86ef2c259420.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/0fc724ca0b6a144f1a3abb1c92851c9f1741b64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/2e25f7bda05de897699fd1baaead85c0d9223949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/94de6ce15127920fe658c45ba5f55338804f5626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/c71e147b01416738f20611a17337395c1edad88e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/cc41fb4c90c5ab2faa128744ce0db6d9b4836a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/d94c9845e1c93589700bc9eaada85199e3193d61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/ea32bc6254de1690878b9f21075911d28ba31ccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/f935c49c3c57dba2e6c920772e6a34c35ba07f48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/2fa0f34c0f609ab263e4b9582eb36f0fa43f03a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/44bb1baa591b235585ac69ce7f1458eeb56df8b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/453416639344c01600cf8c78a2e161bc803db77d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/5a5b8c2def939966c7dbe89d3a99b327998d1952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/732868923d5a085cf3ed1fba228d040e64c9352f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/753cffb2f7ef2bff01a443af6663101d7001d24d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/83671740cc2c42f4f00a525c601a83ce1306f1b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/95c2308547a0c93b8efe32ca5a2693ef94670e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/4d6c8e963d89a23b0c6e22615180318f434f997e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/66aa04272e4573d2a2054e07c01e52bafb57e984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/8447ae4ad906d491e9c696180f645f147212eff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/8d8409f84c9173152134ddf77fb67fa327b50632.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/a19bd0c339c3feb15e6c44309ada004c239612f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/a663ba416e711ed5e51820d7ccdcefafb33eaa4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/c6b1f6bc17df647717bfd67e42bfc9d87f927c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/e32ad125c7a8d855251ab8e6d6a5e1393d562186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/057450a0be221302fdc9e624ea55f115b23fdc81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/0aa5ef578cea8b2a35f4371244c399dafcb4f1da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/0da12815e944e69cec37024eb944255fd8642d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/39b9f48741003d9c806457a51196b743306ccc72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/bd0bee19fa365cbc549fa25a191f85649185462b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/bfba605b17551b19066fa3c3a7438f43b723281b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/e9679ca9817af25d99d805189dd215e3a387e732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/f962384610af8f9259cd950fc8fcb95615fad2dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/002762bff232dfe89546007cf9124492222a11e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/57fa6d071717bf37545cf00a82e59bdd0a1ff479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/79fdc7c964d18860d0aeef7648aa17be0f955d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/c9c2a60fb8e8cb5b92ed207c3b8c20634fb747fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/cdec2e023b35bb7743331ee443636ea9e96f22c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/d0697818167c7b79586258df761f77940eaac164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/d13444ebf5b4993fb224802acf76b0f862bec315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/f669b34b1bc34f7071f400f5c6ad9794aeb44d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/1724eddc4bf98e8088ea79a2b3b2fb590b3599be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/25bbd3c959b24f13418730824912b44a6124118d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/4cb46231eb03f4fb3e3af4fab3a461eb2ba75eb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/5780a101679d04c7090891a75ddff148b3b188ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/8b6dc70953343cfd01b72cc851f46962c7e6a943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/c47a5c86b614ce8e001b3812574578711097be77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/c7e174814faac1fac2dd664399f36d6d6d1f982e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/e9313b3ea14c4ff20f7dc68ea2b4c803ec6bb56f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/04dbda81ff108130c65389013943e6fb3aae3e6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/0822ced722f220d546fe60ff9336ae342886de07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/28f1ea1be85e3fcdaf097230dbd4e88faaab3109.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/2f11f2bef69693f9ba18e7b9ad403dee1ce8c217.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/50c3d2fc586d23737090ae96cbf6457b6dc71ac8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/6763984695a2cfa17e38786844135d6614014f96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/e9274e9b4c2916d50e78d102cce042d4afcce95a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/ee2a748a564d59cfe11cb88a6703c69bc85cd68d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/1ed8907b34b9f93c22799170bb79ac1b2171d7c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/30526380e469c7456a5e9a5392c561b701de1b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/5855271a296e26f227e4c4004144b855d35994f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/7dd274314d9e72475283cbb6c1923bd06548e903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/9ba36da69f73a416f2a64e1af190ad7fcbac8ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/bad316605dfecca29cde8a0e3df3f16130b3231a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/d945d36d51302db458cbaf91273da5ff237c528c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/e297e9bcb09a062fbdeca9e52224418548375fa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/1b9d18b09d3fa841fc3febe6d201439e733ad5de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/2f99f4525b2a8ebf90060b099e8307ee714bd169.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/6e99c145b7042b6bbe09a0f47c33a6b35d088799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/781495da4ba3e34e58737177c0d70c193c86a754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/988689194789e2106fa98b5cb71f440cc87f1a09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/afc060012c59f86268ac0e8a7adab2c0897ee6c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/b847fd1c4f88952e3739919d282598b9a9e89bff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/e61b6dffb46fab8abb90fd63947d962060bfd489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/1b35817726f872364ed12a8fd47cbdad3b7ca40d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/3d9f2943486be52224e37a2bc64bba1317052266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/4b0901c786b9cf22aae7e5f99adf83858a02457d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/8ea9b59e33bf4e523baf99baf537c5ddd7b08f93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/9cf84085a5053e8fef1f603e3edb5c90d06f0dbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/c5c28d7325879839242198d435b2cadc55468953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/d89e546c769fb067d3c956c815061ec0715b66ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/ff67923dbc307761972ab0516097388d167cbbb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/0f8d96e9caa5aa3949fc5bd83042d8d4f2fd05c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/1dc003521ee9116ea0046c8a23936513e0265700.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/20f00f09726d85db6665349a80a13078e5b52741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/352f2d14b935449a9372d77ca19f828e32a0e8d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/69ea1f65ee10f7e3205e987da6c1b674b4f3b5fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/b093e70b2b3360c7333d78ef44af33105467eef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/beffd976e65af7e693794199add6a949970f1bde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/efab9cfc2602e5efb8132ff7bee65af20bbfd7bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/1330fbd97e7e8cb9f267de1385e0e14567d2c229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/17910f18cdd67960ca23c74b0fb98ff90809961d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/5538736a3acf1886c41de52f35aea74334b77bff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/6199e469ab80e680cacb06deafbb339e2c0a4ae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/6e20fc5370b1ab5886e2541a26fa13b787afdebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/c11f8bdff01b954c854182c531174ce779c7946c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/c3e97a6bbd4f23ff9e1e314aae62a120ef56dd4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/e149b0ced407035bb613e4d8b150b52bc15e247f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/06ede12bb09b5876520ddd57195de420199a7a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/10d8ba2900240f617f5fe8ac62265527c336a101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/20cdcaa4f8b3493c54efb716dc72fbcc9bb8d455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/975ab11b74ea054cd64ba9bf676d3f3ece49342c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/a15c1bc4393b2f389256358df1e26b54fc4ea108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/a4a9ed0a8bbe81e8b031adb5327a49813f55e3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/acfdd0a5cf20e8d9ade2a094dfbb46aa05445d48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/edc69acde4062cadbb6f06e16650d80be4a9c41e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/0bc830b3680e12aef2c8bd7618c25c8c0312f16b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/1eb5c207d13ff1fa1b4be03a235f46eb920f612a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/71656d8c230bde1251e9b338443e263be2f67450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/c41d513a4730a4022c121f8c06f4083e3ac32c24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/c957fd92c9f87e93708816c3d51f6213d3c3b216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/cf187e6e7ff2260ba224516fcd959b25ea4574fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/da8ec976ed633f66f329359952f1f5fcf6ede48b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/f7aee0f8c582f37469f6b7605ffd896cec158ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/3b810563a77cf7f54d9069ae70374491da902111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/4c67fbff2265167ec58e461d0b23976cab81692d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/799ce5205817747f8097a14cf40ae1353eda98a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/89d3092541a0ef91e194efa9963c873a4568b444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/8f5b24be3f1c6e2bcb86b41ffaccddeb20639cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/cf908716c28da45b2e68bbafccf7a5d44df8930f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/cfb4af0badedc822c9f9b2bf7b4033bb78fe7662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/edffb084f2700c4b10163261168f4d6dd035ab01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/2f8b88e008e6ad67bdbc482b47efe2f6f0a42303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/8b0a69f73dd031084ae4b184e01704e054eeaa17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/9315720898149816d00fe44dc4c98ec70af73586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/a578c3c93bfa1681e0ae6d11fb3a5a1c4e544fab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/a750f5648773bd833711035241ae188768e0b428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/a85b8280e91465a1ef362af24c1f315b6902a4bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/bc75990ccbef187f1cede944b50599203e7887fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/e5fceef06c86ff87b2bc19d04d77fa16cc655e0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/41b10e098839af13b98c62fed552a22f0c137118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/46bed51a25abf6baa030c1713db53aeabb165f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/4ce081256903fc88b7ab650817f4c1f656648376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/5337aeaf4b064043f3681ef43d44fd63f1ef7255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/798fcee13bf26b101f3c5342e5a172479a3ec79b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/b8993a5e44531ed760112ac36555efb70b3dc1be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/c615ea81556346fc3c04e66b53c3504397e0c91c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/d528baefc83d5192dfefccecd029a7e29e6e4fb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/1103d2a3c2de39e513e9d0ac3f46c3663edbc709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/40811d8f2ef174f17acd3490394d92b775990cc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/44a8315ddd7e30d3ae7d0f2091fe3ccfc2b09ff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/46afb6cab6cc8737d9bdb36c3d994abec4937544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/4e1fa00ad2c00ed59eebd9a3ac7ffa9e2505bd5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/63bb62b75a9176ac2a2836c3e2dedbc02f230d29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/b9e8284952ca40f3d11fdfebe6eb2d0e967128cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/c683ac20944c07a8facb8d80ae103cb48a2a2d0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/0e8841ece651178964fd43ed17813385e923ea9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/2983d8d02879070237ecc8e48d8738e1e4fc6b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/3a64405cfb622bae312ba458930e365f173bde96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/3f173801a5bd385d4a97cdd401c211e90f50a5b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/a311bbe8530da4c1bb94e53fe687c34d2d49af2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/c3856647784365e26e14d2df055a7279465db8b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/d0c2c1f5685f3fad11dc0bb60a2fca5bf398bfa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/d12ff1ff5df0134554845c0cf7ea8fb03c9745ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/110ba8927773d346cede80bd066e55aed26a8229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/399843fd74c96c22a76225897922b94da36ca25c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/5cbf251accd51e094baa87b45f1887e0c9ceb307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/748265c72d486824343d4ba38069c78e676ec58a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/7dabe53b18fc659a89e41b1d24b8c09209de786b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/8699bbae76ba49a2de6f79e79b5e4d86ad07c43e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/c1c7afa620422215066cb55039c7143935245e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/e06820538b554b5a29a2bb16196314afa77acf92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/19a7902e909aa8cb16b8c409ace41c685cf2b424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/62898ffc288e3c038a9da5a083defb4cd5d52c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/7c28c1e45305cadd387d4360e8c832104123ba08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/903cf5368bb24385af2717ec7c0488ec94d5a81b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/a56c3a59dad4c5e4a8ba4194a169681395dc0135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/ba8316bad81bc377236c55e572c901d0efbd40c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/c68ce2284b3ea4ce255988cdc8e6342973f601ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/f39a99f93b7f202052a757e9e7bcf793dde3e249.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2018-13/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2018-13/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2018-13/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2018-22.yaml b/configs/cccc/dedupe-url/CC-MAIN-2018-22.yaml new file mode 100644 index 00000000..a50952d7 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2018-22.yaml @@ -0,0 +1,661 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/04fe445ee4a3c66d325b7677a4bcd21d60b6d4a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/0775271c96a91f6010523fc9d30886653b88bd85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/08f86a1d660eda9be244d5c2aac707751a91349b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/0cc7a6668f632308a51a8aea2a766ea9e7bbdeff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/0f1c5ba95f3d26b701482becc0f78f77760cbce0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/131cfefd9212fa92660ec84a99e97f2a5bd9752e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/1553d2465298507694e710981ec4eb661c07c1bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/5249661b8560d6367f7a7c0960a2514df6e8ca56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/57d7bc7fda7c4ed4ce74421b54b493e8bf4c0a01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/8f57f67e90d6fb0c17427e2510d18922f05c5438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/9ceed74fdfd42a19e5713a35081a6514d8e063d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/b56bf693fe4d422d67bab1a39890caa877c8f648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863410.22/warc/16b55b0d420a034927d58e4e47d2bc16ddd176d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863410.22/warc/a3c4230dd583768fffdc7739c74a8f8d6618fd61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863410.22/warc/ad801ace626fd2973ac530d83169d5b83f8ea434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863410.22/warc/dbdc2d527b299a4fe6005c738f09d72d953b2f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863410.22/warc/e830db61101fce9aa0d42ca58cd4bc4f66f6c21f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/4810acf2c6f77d4da73038ab7e0925c50a703d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/60800f0a2297d18a98acfe0f19a0b184355a9eec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/ba4f939a8eb9d00113bcaffba7d3280fe53ab871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/be4aaf8583800590d50f637f805c6bb76f8dd095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/f81dc6ab3d0f810e4f5d8a71416c028a8009ff25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/ff174c576cb6c7286e6d6d029ba29ba725d6cd75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863626.14/warc/15f0d99be6c170d07807aae86ed320209638c296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863626.14/warc/2c4da7dae2d1ff680c232c12dcb528635abf8ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863626.14/warc/5f0133992adb3402c5df37e63e2ab82b1230e2d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863626.14/warc/6405634958de08a497e8b6e36acdd5afca3ecdc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863626.14/warc/c7339073a95b480e93ded006b5d03afb189e7865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/26e709cd0b25380c0218edc0d47a6723aaaff2fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/5ea269e6cb74339a174b48ca46fa3b48ec4a2cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/5ea4d70d8676f5f15bb49e8e139aec5387ec9b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/74a1e2449a775622061d51f6aadfbf414de430bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/b77fb6654d0cebf98bc8e636c51ae1f4a5dd2fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/c69694aab9c321eca039e5485d44f6a85e5ef908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/09d6be2ef26d4e5a253b978ddb9a49963fe4c36c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/2cb3286a85a6b08bce2bce610a44d1a795d9b790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/93d3efbcf2cabfcb8aa6ffdb5561a945d8b1ff37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/bbeaa539db900a54ed765d5c86035556eacf8966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/c60e6d2dff5939073389f5c941f305d5b0f49b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/fe14dafe458dd12ac6b89d172a09bdd84aa01b82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863689.50/warc/1357788b6ca5b68e4b260a83547ef0f2fb68f7f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863689.50/warc/72c87ee0e2a53fc7775c989454e1bdfbcfd2738b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863689.50/warc/a9dc16479e74d2751edcf0f7af9b81f3dac3c741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863689.50/warc/b08352d4574d0cfcacd962ce98b9480d67472111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863689.50/warc/b47af452e802075dbff4763995d22004f6beeec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/0ebbf319ade7451158aa130a2928f6ee6aa87e79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/1ca4f53d31a90d63da5e85558dcd71b5204078da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/1f71f28ffbb91cc5c51319314296aae67646c52b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/61988cfb9cd1cc0e9fb59f7e1a0288a4bb8a93d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/6c8a94e817f455385e143c97f505d6e6a1a24ae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/b39f80db12c9780ee6d8683a147d924453679231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863901.24/warc/26bedf5d7a624ad9059a753e296141628230f25a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863901.24/warc/325a41c95a0a495b5b650661ab568f9660b71158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863901.24/warc/94207756f48ad56a4061853cc8475ec73d556f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863901.24/warc/bc42899b4b3c3fe9ad0fdbf8945d2e23cbaaa552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863901.24/warc/cbc282399177d71e8a7558b8dd10e012830d4867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/0204a0452b3cc09b2bbe736c67332d3c4ed87a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/171b51b2c510bc7eed821ad436231d2d3acf060d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/44a0024fb2f6885a370aea7e78fef870a7190264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/97a1d3460bd49279e0c28114fbfa16cbde1a4568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/cf7c605d89119c1c2cf65a9d9ef9b3a60e257dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/e8fb14dc380f2459d37df7a9d6c685e027e63a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/5e4825c35a5b8080bc628c198fcc88b8e17b63b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/7fba079b03b3afd5510d369496743ea877affe50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/9afd34fbc296a8e0f32e13c69f7f77de5b251696.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/a01e039a336cbbc3c03e7cd3e49d5118cf3e3b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/a873293a97748e01dd07e4b2bbb63c5935e218bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/e7d7dbf2c6fdca595b204ff3aab3dc3f708cb0c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863967.46/warc/10e5876f2359dec01b6d6773596f53f67e190b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863967.46/warc/1becf5a08424832cb96146906345a26de0d4f6f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863967.46/warc/9fa4a3cff294346ed03d84b176b2ea4683c54b94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863967.46/warc/c0debd3e7692c5bc676a03c60bd2bce79526c108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863967.46/warc/fba2021ff33649950b7f1e315915b3e3f49e7e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/24ec183c81dad1824032c41ac4280a25f3255392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/30d7a09573b42f628887d56dfbd421c23147c310.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/3a4da7914b1a158904d18a61c7cc5253a12e14a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/47b65e75378eb4ed92cc8b03e7903a2824ef2aae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/62924bb1dbdbce03f96eebe5a9e76c9cc9c8ddda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/eabb886fff651585362d80f69f6ebbd8546ed613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864063.9/warc/00b3be1e1edf64dd7c16122d17bfa3b0addd204c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864063.9/warc/3a337ebddf3af8727d151a24be035de8ce2c49f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864063.9/warc/3b2d6413bc687ecc067cfb8c5eb18846f65f67dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864063.9/warc/956c13c8d152b8f8d8a07b758c47ddf999833850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864063.9/warc/b826d85202e14004e272482d15a86c22fd86cb01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/0fbc31f92ab92b9181bb0456efc9e65f6183f5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/3ab64422010e74689fde21082927ce6edc0f9483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/4319952b55f317d017692bcc418bd31cf25dae0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/5b06e8dc39290f3ecb4160c378095696a2f74a0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/6be329e058d4347271f0fb5c39fe6d3d9a3a3889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/aa05c400d852273dac996f8b75ac64cbf5853fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/3d70606feb1e13f34dc6e4e0f29429ec4eabdcb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/8ac52d57ddd3909cfc391c95fbb1763bcaaf85df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/d115073dcc3be3b6312459470716afa85a7e5613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/d173241354c7d2f93e3c7558ec64aa7b6565aa0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/d9fdfa93af464ffafa236ae1deb11648e121f546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/f5b065d344a2dd5b15d465b98ef59cd1f3cab39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864461.53/warc/30e4d4455d774cfeab80436598ba9cc0ae9369e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864461.53/warc/382610b3fb1da9767619156d1440fb8a35729a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864461.53/warc/4d7d116b14a14d47968c91d0f6cb0a19b094c13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864461.53/warc/752546181cf116500b89924c56e82643e1d0cf05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864461.53/warc/88327884034b2f368d789d0f2bbd1cc2d36c9116.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/1087b6547752786033b47f5908c8def965c4b91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/18217688cc8b3a67eaeab9a181ce42c50f2e9050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/6df2ecbc1007dfa65bffd17f3a90c11e033e5e9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/8062217d5f3043fdde28c7bd59598b9f0a3aadfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/9021eab22726818fbd94ae1fff5eb4ca4eb01e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/c4450050b464c084058976834d532e656e37ba90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864544.25/warc/1f6cccd118557bf1c9b53b30121278556024a48a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864544.25/warc/39b0b1d83acaa5a297e18a1828b1f129dfceb77f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864544.25/warc/b4e0d2a4a550c3774dd2d24bafa7660dfe8600f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864544.25/warc/b7e551d610986a5a5deffb6a67175db66b721c99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864544.25/warc/ccd93dc5c2514f9f93df5904a8592836102366fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/0969005909a018b0aaa35dca5954fb2e95bb5356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/21ac169b8baba84fe676e954593bb86b274ad678.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/49ecce85875487104ee4d9fe62de273ff8306d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/52ce3bb25af930d6c63f734b587540f948d5e85c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/84348b969edda25a8ab05c7aa613ff6ec9cc2c2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/8bdcbfcf84d3099c63ee83e2d49b3733932ca133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/2978a2e221175ed75887396c3667312f053bf40b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/3526c5a9828c0bab5d62cb96fd8242bbbbe7cfd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/5ffc8b2a4ecedbe7ac2259954d80ae5e6cda724e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/6523fd53998515b6a27facd75cd6d7c42e4b327f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/6a0d4f2616cb6745b56b0554aad93921804fe1d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/ae4aa4b5dac21c1a92c118b746931e78696a2567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864622.33/warc/21c03441d68e81d2899e1361b9108c6cea85b42d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864622.33/warc/34f766e9ff1100fa100f2c9ae49d45685e725269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864622.33/warc/772be98565c5f45f834fee6f74932461b483e134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864622.33/warc/b942a41562c82fa630221fb1732b1f422adfa6c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864622.33/warc/e13a12f0aeb86163e26a731c806c1430854fb510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/051d258befa3c35729fa73b303d0a04fd3ea4da4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/2493e4dbaf9ec4cff766131d19b795f5476c1b28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/533b868c960f6b7d1d04b878cb689786541ac5e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/76343430989ba67534662fc2956b8df76ed9d2d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/8cdc196663e3802ca3bddcb1c6f8b5c50ddd1f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/c7811821759979615609688383ca4219cbff6c33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864626.37/warc/0a76050e8bad3857e3cc9aef7792773aeda5c324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864626.37/warc/5f3a9bc487e22624ae597b541af4f72cdc66584d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864626.37/warc/7782f19aaa5395e09ee5e11a5268082e494bbafc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864626.37/warc/7a25e798dd02b354cfc1059462c127a2ca0a1f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864626.37/warc/ef40bd6d3be5553c0107c84a0b1de4b1d0cb1084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/06593ddd9d8c487520dcb23ad1a328d2b4ad4bc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/091b156e57d22fc78d2985351b61468a778ce2b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/0a20148d070accf899b973ec17b5e4e899a7c2fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/28b9e52e4c5408ea5717078b4e1b1efa42588dee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/9f4987326a0df8748db7d577a0584e2df73078fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/bdae23b9115deaddce1385edea45b232320347ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/3cbab7b4b731e1849181c376279b42d2e0b08b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/92c6f2dac6edc852ebfb0f2bbceb8d1c1252cf58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/c004e5d26f2511821f375b7cded9ae513c8e61f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/c098b2d0edcdb8532675b8bd38b6a8a0ce524538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/dab2f74b86c4ae32dccf69e512009aeef2dec7e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/e2fd6292f1a4ba4b28846bd7f35e5850942c9617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864725.4/warc/2289495dbcb78e467939fe18bfe2fafc7f871cab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864725.4/warc/50a958c769d7f7562892a95ba94241c74009d5bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864725.4/warc/62ba81a28f17a6c58a34ccdf968f3ac361d3767a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864725.4/warc/9b740d00a0c35fa36de4299dad78b547c19e12f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864725.4/warc/bd1c3494cdb052b442f3c59889c3f250cc59a33e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/13e6722f82142100e222040b167bde1dc70fa13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/583642be54f08e176a4a402b7a02366390a08a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/5a1d201905702d924a7f5a0d4ef7b3531e2e3ae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/9bf2166a5315b6ce818b9eba6685518c5588be5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/bab36d5af6f33ce5e364607d00bbefab67b648d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/fc1317d17f8302160f84de7239d0c7244529cd2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864798.12/warc/0cd2f7b9e7a4bc371e2f8a76fb726e4a6de678a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864798.12/warc/47c853ffa6db03d29e0b3076a6b374b0f8f2851e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864798.12/warc/853383feb2a0658972a514a24eb3488ac3c9cea6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864798.12/warc/bad63ad7a635a76d983d12573568cbbb20f3a702.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864798.12/warc/f267d1f958d2d50bd4ff7e504e39bf093f7934e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/05d085f0b17673a78bb1ad709edd9b88cf14489f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/45478f559562a79f8dc24301cfaad6718ef02c62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/65e925c3fe64f353e262006d401395dd0c4075b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/764154d90ade14386477418bafaaaaab940056ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/c0cfa29b8f921c907e0839ee0982c1972e447f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/e985bf6c5e44ad99db6084b7688568896843ed05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/88d33b486eea2998d1b3f4b8d63c12634d742516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/9de4e2487c66479b86f706e12af3c5de202d079f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/b9fcdda88658f86ac042dd2133cd49f05bb494c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/ca3cb6f0e0eccdd3529d4c4d3b778d7ad0f26b9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/e1f8ea496939d2a8aee74a146e84743de00e37ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/fce07a9ba60195b8f42b4a98f8e3d6e6be93411c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864968.11/warc/03f6d515534c5a3979933186abb1e3a2b3b1898e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864968.11/warc/0ec0e4b497d7433ab601f5050257447b8aaf9553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864968.11/warc/131e5e52c874a7ef41cd1cc3393f9f48746c4d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864968.11/warc/151b43bc5e5d7219e3cbb13f55d497561e2a7115.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864968.11/warc/69f843c88a82929225d136e083342be992b9f8ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/1204a8dc8e4eb2557111c4ba67396f15bbb3bd8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/4882c841eae7fa4281bb587b482b34fa4dd74871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/76465a85817e58b5374165e17b0d72fb144c393e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/a8de22d11946acce19fedddd6ce6c2a22e7bcb10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/d9bb1ee9cd66c4abeda294cd5a883051322bbe16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/f167171098b084343bff1310a67cb84fa315e5f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865023.41/warc/2fffc24b3fb6b6886c0951162fb6dfedb4ea6185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865023.41/warc/9a11925be4a9f58a526626c3a81c78d690318619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865023.41/warc/a23433ad99600f6157c4598149f21c66f6b3d1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865023.41/warc/b287b2a8e4c39150548a272af69a1295607236ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865023.41/warc/d18b7a7597c1b53c09e04ae2ebce0f8ff32c82ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/600b9540bd1701a7e7e6a8d40bf203312046f6f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/96672d01fddc3767da9bb986e115ecfad4581cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/9abd58254271f13673bfb53b48b436e922f2a1b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/b248a86538d7c280bfa786902548e1eb71671209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/b9e723c65b634823ec71a1511653b7404f051be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/cfbc4464bb685f149b56bde7cbb85cc514aa5cc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/0132ff911012681103f155bf5752337fac8069ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/540e36cf0923e40203291637a1aa77b0736615ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/7051dc5bb791966ce771eeeb9b0a7ff475760452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/748a77281693c0b2c108c34fd7ce6acf64d04f79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/d45bf4301e5aa3ad04ad5241a676b2c74c246834.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/ec54ff9f0f7b6445ed103d2efe04cb83ad55842a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865456.57/warc/0211a463fbc0f2c3ae8e5101789d2a677fccd0c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865456.57/warc/30439e4128da4fa74283d108f2aec44d42ee6493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865456.57/warc/974fb3e980ca0fc42804c4f7f9f278fa97b3ae73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865456.57/warc/c73b9b193e53489ef4794d4d76d7fe8595bc6e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865456.57/warc/db687a3123df643213c64fddb7c4aacec765d135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/27315a247b3b76a32a8dd58ef7a91a65aef1930d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/3d67d3229bf26e9c4cadeeb4ccdb344c8ccac925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/774df7d3e00bdad86552f3fbc9b310c242c885b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/7c7a3770bfa82d00c47df2e010228caa5728e47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/93002ccbf16565d01a842d736758b5deb48aaea7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/9bbdccd1a9e0aeb1a779b52cc4a2ae94e1751fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865595.47/warc/13ffd914088c9f3160a87156010040138db9d5c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865595.47/warc/2123e10d5f9a48ca960cdf2c25a52e793ef64102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865595.47/warc/4db5069f072ed575ba02fb1fc6331b8a34a5ba3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865595.47/warc/64358bc906cab4783993a20969a3ad8fed85cf13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865595.47/warc/fdd55bf8e8348c081eb972cc9a36e62562c8942d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/227dbeb3dc1c40c1f951a9d1c37a81ec3d205031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/c5a03ac1eb0deaf70108e229070d978394dfd34e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/d8930b01b3f6e2eaf81c0afeadd20eaaf210f007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/db37c16f1eccc08fbaa272af91c509c8039d886c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/dce1261f630abf026114961b6c949afbe19d0cc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/df042d412cd65692dd0ef68ac2e29050183109d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/08458bd18482906758b25ebc81418d22f3d82b14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/2cca4a57fa10195a8c30fa5b70f7a2b8499eadfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/42a8527f3d7ca5b5aa2a3b02601f115d9843822a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/793c1887e33962778c2e2875c45a2bf6d0775b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/a84ad582b2ab5e0f2ca66da81454feb4fc1861c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/b11d644dd398a2c7219f3d9d568204e6936fd832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865691.44/warc/3a2b7b8f2238372d051779cfd43cef535f6041eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865691.44/warc/699ee241037c95dda22b3c6d84810a85530ba033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865691.44/warc/810093ae01274094e1719fd1f1eb288b394e7bf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865691.44/warc/8b55ed7d6028f266f49afae5829045a080799a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865691.44/warc/ad079e02609e051c69b18349811d103a32f9959d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/1a424d240cf4bc9febe1395b6340cce6d1390024.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/2e8022f70d0f62cc20133094ceca787588a488a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/68daa99e1671ba32397d942c476763694ee0b817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/769233b948adb8c321a6240ba52b16a678582d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/c06f357d4d47866c6ef124500ebad95e9b44c176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/f627201c391160a65a0b2212ca39fcfc7c599d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865809.59/warc/2dc9c0ef4ffa4d7ca76385cac2abbd9bb10ef8cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865809.59/warc/4c32d05b299e43a910f43e293ae28bfc2bb71eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865809.59/warc/6c771c48b1214f2041cb2e39cb140f8dbcf83d00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865809.59/warc/d07f70ace07a55c69967e59386d03d55d47d45a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865809.59/warc/ec2c3831aa7d2219288dc3b76b5e872e40532ad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/1474ea9f0c4535e28a567346cfd2370de262b0f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/2c032b16242c21fb42cd78573c935367744f8ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/50bdb0f1e3922562c07b4ed943bca6b929e7e0e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/c0942ef58576f0f89540c2067a4e71cea4c8915d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/caf2566e5f4a01ff3027a88160a3bb6633f4ec54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/d3bfc7af64939e82eb0d888b6562c1e4fe25a550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/3fc2d1b95589576f0b315fd3d4272be1d46c1515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/4108a0f9cd5c1d19db3d7375445e8bdfb1cfae2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/500238763e3e70f508f3dbc65b491cbbb7036068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/784e14efb4e7227244e7a10b8e94073ed100fd03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/789e20da50d31ac0b4628042a9265c51025bad03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/ccd937c39cbcb52f51f436f68722f0e13d680986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865884.49/warc/008351d2e8720c815fa66d2b2f0a08e149cf6992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865884.49/warc/1fb36052aedb69f2f925a7a064e35b5784cd2e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865884.49/warc/4e9d4aa58275c0a9cd14fb58a3492963e49ee370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865884.49/warc/7296a4f21477f9a0ad809438550a79713fb5e1d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865884.49/warc/f20aee20ce7dc282e8a09686744596615434e3e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/24179d14cd04005172131ea666db3791287d7c76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/3fd343de19d4e3d227271eae357034fcaccf2e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/843fa506898034e8f7859cce92c49fab1888dc77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/97dc6a66764f463d5cd3d7e5401b47e095479f78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/b82145cca3d08bda2f60562a929a5f27582f4b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/b8db1325ea3b00fd4ad82d59f797ede6b297aa1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865928.45/warc/6be6116672446a7f847c0e702f56339711d8427a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865928.45/warc/79206a13f9f08d875c8ba407605afa4b00907050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865928.45/warc/88bd97a0a72c617bf6bfb17e41992cd23ecb7ffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865928.45/warc/d6b3c17332aec24884bed7a91fbd2e5429fd1c83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865928.45/warc/e4474524c66144a87a1acf361c9b73589ab205ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/03d87d04fe601e23e51930219d6def764b44c1bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/232b13125feea15dff411a5fe916eb5c4b100ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/64d1a82fa47787789c33ae838c8ff6edee413bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/854ba702dc1b53d9ded10c7530738b6d3560e0bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/bb7690c7ef7b5f7d50ad34960b5a726c85a23883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/e65357c2e275c9bc6f235938705c5f8bd8763b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/152da02d34e525adb9aa3c2000422dd3499486e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/18d0f407a229ed02a4599874acee35e0935fb4f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/4595d40bf9b0f94b76dd3a8aec3f1fa222a4a93b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/705210e33a211e21503fbc64aee4cdee6a7edab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/7a848542ca891d91a8276f350d7b9cd17848f599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/cdb201d81b63a4ead5fc97b25f2fa176f3e25e47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866276.61/warc/32393c7e50949e4bf4b9319077f9186b4b4201a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866276.61/warc/71232e4f26b1fb877e190f7684aed0d2f575f1ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866276.61/warc/9aa529f7db7c6f3d274e9c932670f67a5487278c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866276.61/warc/b9d4f84bdb518fb556be8bc9a3e109aa7e290af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866276.61/warc/fd8f1676415bb49145e70cda05d00408ceec9dc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/0f5f0a18ebd2489188cbfb631445fee41fd1be3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/860205576a7ad8c308fe5a853125c417318fdec8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/9acf9e90964d7232917e178ed472ae86121ae8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/d977991ecad75a720f9767102b48ed32b6d34794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/f9c1a4b285f0b02e740488dcd4f4f913d91f6136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/fd11bce1d72cac715f03bd358675cf09327dde0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866511.32/warc/11778f8364b03f536689c1c9e805ba3e5448e06b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866511.32/warc/27c9932b30fab9efcfd562b978b49d8da0fb1da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866511.32/warc/4c8c259a48c8fc97e744c42915235784a811da52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866511.32/warc/b0d6285b3352af283ebae6137f7474cc13a974b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866511.32/warc/d1defb7daa2df613500039ca5c9dbb1c54fd1a6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/489cb7f8d65d90db89b68350285687c574cdfeab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/4b24a5feaa1b7930e46992378b4104e513595963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/61e72813823c8e239ad8374a0f56319d2d664edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/9c6972dbbecb7584a3383a60737ef92cab726a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/b0e7bac9e1ae54ab5d5345eacc247705eaa4110b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/e8e94b4b1de1f4ea118b5d506ce3cbe565adb57b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/0fe1b68c863197200d5ca153505f71426e1558d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/1d8a5a380029ad6899e211a44278d20d1a098445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/4aca7db7fb9a3e727a8303352223b47890284eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/5bdb0281bcf755c8c2f9af24ac659a1f13a1a775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/89bdd89fbf6ed89c57780bbfeb0b7ee488b7ce2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/e9c7ee5f51ffb529384b3c622fc769b09364ee37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866870.92/warc/174d6d8fa883076ab4cd80b2f026475f6e334703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866870.92/warc/40c2aa60b119eb50e0b62f511a7aa0b7e7f51b87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866870.92/warc/4cf8a3799993d68ee6b92ad2cf46b9c367b194f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866870.92/warc/b2146ec9ead41534dc6b964a69f28fcc1de0e3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866870.92/warc/da0f9f189aed5e3c344149a83773157003ad5d6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/231f5113827bb2b2b86508fb9548873e87cd7d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/8b49d83eb0d6838be751b34e3f695acdb83393ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/954d4360cde7097ba7b08be8df36b77e0ce0dc56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/a67ab7210a67f874327768efd68d74386839f675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/bcc772b9604fa2df2d1c6421d4ffb36ce79f9505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/f72e0587ce7fc9bde2a4b2515c621e3f29cc4a1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866917.70/warc/34cbfee88a1bf115e4f7ae4203f390f736069d71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866917.70/warc/a91d64c44d513227d4a0026cb89208b6ee3bf187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866917.70/warc/ae52cc3fd1241dd953e6e4a2a21205cc2c701912.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866917.70/warc/cb2c7188b94b6e82adfcbeabec6ccbe9496d079d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866917.70/warc/f6b44d8ab189be9270403e4f675cc123eb4582b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/1ec2cb48d7a760832e3b76457863df5c4d27afc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/264c2521447d217037925ecf17e6c445f46a9171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/470ea63a31eba8bce1efc3d7db11d231bd53412a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/cc39886f2f1dc645974cecaf49c2657712b0be62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/d9e85d41bc56bc2e75587cd9d9026ce501de7b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/f41eebfc6e735529df3461a6a75e0c15135e2f7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/84652109b35be9e3bbb7f639da1e24a6d6f0ab0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/9350657936291ddb44a2f1f92845665c4b0d05c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/994258dc1149342f821d7d6a0e053643a1c28d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/a1a83bda1c5e684cdaa68f84cd49c66fb7a42df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/edbe70a29ce07c8db0ed8830e57fb0f3ab5b5dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/ff3deb4d303895407a3fc590704afd6697384811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867046.34/warc/1c41845bb22d38a2b4191d371ea3639b2dd40b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867046.34/warc/1dc818170759381f6ae469d0f28435c504d6c393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867046.34/warc/759a063a524c8510b6fdc3a44da4e274dfbc295a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867046.34/warc/d6916f2d38ff74295748b4a482f143e4f72b7ff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867046.34/warc/d91672e3ac2af1eb83c5cb9ea6a3478e4a9491a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/3236bf224581b73c2774b66ccea105371dadc2aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/4cc3e12faa56e927c4f026f9a1c1030818f9781f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/8a5be70f4572ec6269d7b9a20678035f91fab402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/8b2e41b864a48662534062dc8dfa66b054b6d772.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/9e10ed550ebac8408c40b39a586d5607bc4f2007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/db7ede25d98c857c4ad45201326567a1ecb471d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867085.95/warc/0a0a50016bd3a43d1c39d5c3ca7749a8365009e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867085.95/warc/0f6d02c98c2fc9266e576b11709bde5d0cad20ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867085.95/warc/b5e2904a605b61b0a256e51247b949a96aecbb8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867085.95/warc/dd553d1f8b796c3b1648fb6a0f2b87ff7b212aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867085.95/warc/e5ba9dc4aba2b51e0761a300c4f6c7903d521623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/0a777a6f2692f3d271f482445a443c353664bb5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/15219f1f3c4f0b6a5c05555a9a8c1234657df94d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/18ebe2851ca829dab5583b795b65da63a786b892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/1c3f585ddfcbf4ecad789f0005af94891536e540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/812d1401a0f2858c68eb68553fe64402774b0c0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/dd0e48901b02e469088e9e40eb8eaa38dd277050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/24521f7c7381ca3c0eed17ab5d277b361b95c7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/3e4ee66d15c565f7d349841eec4db88e0d5f8e32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/75aca4aa6664e0b496e5f45539259adbc9a85f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/9f54f69ea2cc80089e7e0a06835cc7c17f1913c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/c711e99511e2e06ff59e57fbe08a324423212f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/c9efe24f617e2236d9ffe70df1c8c0a7b437574d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867140.87/warc/1351570d32c428bf2a887a1a3f51cd710c7c1680.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867140.87/warc/8058fcc6864ddf3d142ab830fc36e69152e564c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867140.87/warc/864a3fee8d83ef4d9f9e47e77d1d0d8b6c2f97e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867140.87/warc/e798121a9898e08d0a00ab5f38549893c55d1105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867140.87/warc/f16c7783e384c123863c8d0b6a83595b9eef404c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/0fa3babd0c9ac1c5238c872b83d64611522bee11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/348e89ab18dfc950777bc16f4b2570b66e339cb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/49ca1e278ffd09b46a6b0338e752390ea2e984dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/8dbca07080a5a541a30611be39eab898fc642c1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/9f5600780366ae9a0dd80aa83a9579d5e59f76f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/f8a1ee67b9632104c28c322b66cfe9baf9af5e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867217.1/warc/6a210e3da771a6fe1e0d6137b0c87d08eb591d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867217.1/warc/9be4027ab49a7b29bfc7aa2e5daf33c1edc44b10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867217.1/warc/b4b45c802232ab4ce187c7c537514f5c1a529c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867217.1/warc/c51011bf7a2b9d7bcf0546492b91c5a2733681a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867217.1/warc/e05bbe58e07c605a789e512bf3de91466c8962c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/7969f534c04a6cb497333071b237c158b43a9191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/8dd4296f841771dcc871abaa702756c75a072be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/cfe859f70ad0f2dbb4b5965e737a00925fc402ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/d321e39d9caa5ea7352d1ad7416c53ec5f9eb821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/d5a6ced4cdcad9a90d6c4c16a23180281c255f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/eee06829dfabd0b57e375d16d1717138f97db75c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/2ecc4cd2fc63ecda0151b7589c8e51c874fdc2f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/3c97a0e6f5b1dc17d367a2917fd9bb2f5ca8d14f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/3fca576141818f16e6898d74e301df3a9cf353b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/78a686aff16afe1d041ca371054d8103a0abf055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/7ca759629571d61ccb66994f163e90ac4bdea8f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/da55f83919b2bfb7a014b60062a82150a54466c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867277.64/warc/33521da09ff8393fdc92fc465dc477953ffd29c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867277.64/warc/63f089a2490f7dfe6adb6890919d80ff7a34d022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867277.64/warc/64432ca015a3cfab80d613193b2e2b5ed40d8c19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867277.64/warc/9ee0e6bb21ef3b8956fb3d5b5aa7bb8f7ae950ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867277.64/warc/c321c843b72a1d43a94e02780115ad5541a9a431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/0630008505852f98562f973ddac8b56ad737b3ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/7b37019db1c547359f232eb7fe12bdb9a8b75ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/a00ab411864a90ba9b3cb49f4b051adef3d5bd1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/c489c28bedf99489bcc4026498e8fe20e9edd760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/c6340e263508734c7adc90ab5a55683674654f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/ccb4e9bf0738b070783678e32d7ebaf3e7105cc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867311.83/warc/142f688f6e2fdbd0dd1ac7d689579f9f93e808bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867311.83/warc/644982b84ca62307cfd2d56a03723b1da77dd59c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867311.83/warc/6689a47150b7d3b5ce0e908b95aaf40a857487e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867311.83/warc/7faa8cb6a9841b75b6093065b04a74a028fa482a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867311.83/warc/8e560458feb9fe6e907a05e0d25ee27643ee62c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/14fbcab1fd1efdf9e4edd51baea0140169fb9910.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/32e54094ba419d618f6faa74f4e5512a0e857318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/54da2bcce534389a8ec2a75f901b7a0975346115.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/9b6debcb4db6734da7aac98b95e50ebfdd563865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/9c0c3a182cd22d31f6613f1031c00885b77ce66c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/ed357f7c842d12c1f05a6d08fa48237a412702ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/7f1ebc225ee407733f7ba8fbcb3285751b8bf3ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/ae227b710093591aaaa8b5454183f633241566c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/ae2703313c9fe24d54df465814db01ac843c6a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/b8c2865ffa2bf6c4ec8fcd7418baa037d0c3a818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/e585b092a5684d3e4513d2ee11b41599d0cc0aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/f0095358d66ba7bcb070b8d053e81bbc99b3d264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.71/warc/26bc81e2f408e22b028bc6e6800c76f4f2c3cee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.71/warc/3bb9f557f2d514a32854bd5981d28f36033aa54f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.71/warc/563b338542276170c2b86f0ec08b8273390a301a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.71/warc/7db5ccaa99221a9975094067498b78e7ffa14b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.71/warc/bff87a4724a0a25f7b163cbd80a31f69afc6fdcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/2f2f9f646dcf6066732e7af4955fd645dbb65fc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/9956a3815f8731d248c28aa10d1de88639cd1f5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/9ab658ec3f614311bc8f5f1861eae924050257d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/ad60ae9f8a8afdede12f3564dc8efe9d785c4695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/c8286b2fe39eb357e8987b39fde272748b9d90f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/dce3705e202b2411917c86ac921f0d57452a5b76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867559.54/warc/6847de9478c46211f3afc5fe0ad18953a23fad90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867559.54/warc/92eb13b7c199b36372daf757fae2c8b82bb97aac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867559.54/warc/d840107b12fe8d846b07a41baa76b0223782c459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867559.54/warc/e0424c25698e4c20c65d3100268352a8bce94707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867559.54/warc/e915e07d7ebb799c5c337757046be30c853afbe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/01aced303feb4b7ead009fe0ff327edd77269e75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/3ff9869350b8444584bcab71564a50b9868bb9ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/5c6ad8d683d102ee4952b9ac99674ad45d9778e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/9e404a1aebacaf866c0456be888c5861935d709e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/c308188fdc2ddc3ad6703f7af4bf4d5479b6fc01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/e3b406f0a94a305d82e9ec5e5a7a026a0a53d565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/311312dfb093b0ae34032b2d85a0fbbcdc2242c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/6c5e5876ce5afa4b9870bf23ce8a635a8504e67c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/b59967c70e7498a3e6131efdf161d57c14d61b40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/c7a30cb03a02f7794924da22bb2e0f66cdc25155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/ed8e50e788f3b8879fe7ade9682a5deeec47356b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/f0859628caa3cac4317ecb63b11573a95e6455a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867904.94/warc/2c32f8adadace0990b4af6c4c900d40f3fa6d2c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867904.94/warc/4575e49e0169937f3b26d054ff1b07adb6cc13ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867904.94/warc/5c4550df568ce30047ef92ebca644a364c5416d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867904.94/warc/708a07452917e3c1f726f63320c6a76811b08eeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867904.94/warc/9e856b5ec671190ba661ef883f07ebabcac1d720.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/00430900400f06a8e74e9ad47559b11e2f3d3ad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/004bbd49b6f5251b0c847d90d60917281c4f0539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/03aa265997844d9bd8dbdd2a059ab3ee4c21aa51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/5dc47c2acc76a924e4342eba9ecfe99104ab4819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/7b19276b8b5d9dbf71ea7fb7dd85db34de57c5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/faaf4c17db498b9571b188cb3c593c96ce8e4fc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867977.85/warc/36ffc296f37b4d1a8b18b6ba9577be498e0c5886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867977.85/warc/6bd4e3f64e989e80810384222848c741bde42fc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867977.85/warc/b3071b6f1696025c1b98d1a28a1f8f4db8433bd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867977.85/warc/c7c1936aeae5220e456e538935a6f313033936de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867977.85/warc/fb44eb7997b06afdec3b69f0c1e3240042ee3160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/2b6bd55f09a9f9f9853869d06a55b30d2173a81a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/399f0e275e47f1e82018233460010c4298da71a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/6a6a65af2fa7a4e7b0d3e2e5fb14c8b6ab25fd93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/aff7efec011779e10be9d8171f20a4243f5f1bbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/de1362d86fcb75abea4982577290863a0ebde5b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/f58b51db35490bf9b8d5025ffaebd19757620717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/048dc3ceacfb3190a8bed4d0f40773e01460132c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/198db49a0452c8ffe5cbd81f39cc72ffe97bd7bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/630cb1e33f657e0aab6ecc57f28e9168ba05fb3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/c121173a93caa74248c4c6aeb9918af96567f64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/d08303219654d297230ed0f565cb84de338a3326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/feaa4359ea76109f636c344b0cececa64c3e4490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868132.80/warc/144036ff537efe0a4a289a67c6fb5fa04b1c7165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868132.80/warc/168b9d7c74b1be15ff7cf945bca34f3decc08fd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868132.80/warc/6e24d737f3ef441a5999e8fb19aa0847bb688b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868132.80/warc/cc7f90549c33e130189fbede1dc15a0f77b66136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868132.80/warc/ec3f7b5f03064841d579f2fe6a02580dbff2761f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/0bc2b6303a815f885fdfbc108770145ebe48d4da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/0f16552b574942a4253695cb99807ac283439fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/40b2066284fbee894d2c4aab12ceba28d761eafe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/7630b878b4e6f989925cb82482ae7543cd84d3a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/dc3438407bab49afebb4a5a9efaf5a9b0ae98ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/fded57500b6bed73c03cfb4d7cf578e344184d00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868248.78/warc/4405f58603b7e7dcff2781d7e8a5216d159624c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868248.78/warc/4cddf19008b1c39b399566e6a786847e1c346ea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868248.78/warc/782e1eb10d8f66961650ae575d87b49c31b53d40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868248.78/warc/9bfabd8182fd5aedde25faf020dc025557a916eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868248.78/warc/ee4b864b97ee285ea4a9c8591882402c261382c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/05fa2f501ddfc1f8ed019edb62aadc593752c0d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/0a70ceddd90e0cfdb6d011fb8ecc4145206becf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/0e4d3f2bebb942b288e0db372082d5332f9d97ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/11a92a4d467c20655ebbbaa9e9a6ce76f66191b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/9daae0a3b2d457c7da9b86164b7a0f98a63956b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/af87ad34e56f40bbeb0374b394a2bd74e85e736f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/10a094862de7964912b0e78443fb98f3dc169976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/21d9d217cdc485c8782064cc2600407909e8d617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/22da84be8621e47ef7eddc5e4cfa35b46f1fe139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/4343fb08d567a552a5da9f92780c61ea96c33e9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/605ed4953772c7f55ae4a2d15f949a65db8d0f0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/7d2044f057d1fe3c3a24ca789bca0d586bdc6d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869732.36/warc/21205a6dec16c8df124812a42eb70f0b885696f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869732.36/warc/8594abd5502f5d2368e89f19c95884ef6d841631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869732.36/warc/bba968b7e1f3733f187109fece06f7e09f408bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869732.36/warc/ea4d77c09ac1e20bb054f936c3800cb5564bc57c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869732.36/warc/edadca870fb522e3c46949cfbe99f729dd24f5f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/0754f2e2846cf723c0629ee5e0471bacdd709121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/1369f65d0c55431b6681a4c1d878f342608dbf1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/892bee52ee2512f1eb974f28cb9ccb33a1333a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/c6bf4610d76544e0f5ed08e4e43b67455490c47f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/c8f91002f77c8b346d246b22d7b731d2a9846b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/d60c4950007d8e228a12df6c70d93c5cb43cc255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870470.67/warc/29306807b6ed9de05f3cb324d39251309223e3ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870470.67/warc/4e5f0c979ecf09f981e3b2a3f2ee21b96b7411d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870470.67/warc/59fff4330f724b4e55772eb106f347eb150d9b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870470.67/warc/76457ce9ec52faeaff31a866d6cbd876ffadc5b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870470.67/warc/dd5957b8406f2ecfa45a5eb085a9d9de97c14727.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/1d90843d80dc2a8c50d7e18130b1aaf907de79ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/337b2d993533cf6dd7c9d27a01fbd4ad6b983afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/55cec946a3cb644ffbeecdef7b220322a36b12cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/cd6a1db0cd823084546a3604f0cfc0b07f4b1b6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/e4c0f195b877d18db09a3b67a01317aa2fd7361a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/ebf861b6addc76c091ed8c4f264de996a71f53a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/113e9331eacf9715b9ffe02c489dd5f7d27894f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/1cbb08e9ecdc7d5f7dc69e7616a9b5a1b06e753c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/517011e252edee02720623cf60927235726940f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/6aaee9732f667c9f7e50509e6af50de996d22386.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/a780a96f6d27d9dc60f9a77965dd500b06c976f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/ba41010f3bfd2963dfab12db33c18182c3b19b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870771.86/warc/402cdb9ce74c29aece449a8a719f8fb934ef8abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870771.86/warc/631966956a89301d60790de980c4375c8bd9873b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870771.86/warc/8018fd0b6ac0b97850771ff2682be14e060458f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870771.86/warc/df870987f032327018b3bf81f9e3ddb25ff10b7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870771.86/warc/f2b3cb2d968dc5fc9693ed59e80c2cc2804d7034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/2d54f1cbc01f6c16db02ed3865b3edcbabe52151.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/3a2d975694978ad122806bdb7ecdece8b16c177e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/754b0ae43baf8f74a76a20b728d5eaec2afe00b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/78a5d0887e105c57c6cad143776b558ecb1f4564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/cbb199c0650caea005e7256cbed87b0ce5ebf6be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/df5b277454487e10519b0b990e3d70a445278693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872114.89/warc/386f3fd289ff2064cdf1d60a4d632bbdd8fec67b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872114.89/warc/3f00a99afd5d980775fce27d4eca6eb86a0b1831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872114.89/warc/9ca14283ac944393686527918749a5fd8b82597b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872114.89/warc/b322100da41345ae6222a81120b788666f641938.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872114.89/warc/f189857b84e075fb1850b7605a86a85008022368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/323b00d338fce95c3236b805443624d1705267c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/6890854777f2b63f69da19a9ab3d59ccd791cd9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/6d3fc1fc70d0a4d47cfcc3fbbb7ccf221df5cd14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/825ddec73d9fe51c19c4ab40154a3da948955435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/870a7747bef6154c95cd75b2b09991feac4cd44c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/d0bf00442d0eecd5eb4e8f5f017e1f39c2d1f37d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/158f28a0afa64fffc70c0133a39c66aec29211ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/17009c45c85b30c7cca304302902b75d9f9be828.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1732754e2c604fee5aef8a3bb33f134deb1cdca5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/22b21b26bc1511c48ebaf4ec95071a0abeb31aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/2303a56bee5fe0bde2d9088dc0a2384721ab7b90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/231deea4208c31ea436ca923166d771d7a2b3b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/259658c25fc91547927813e40ca66b95f03d4721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/28a6e53b0d8dd618329a275e98ecc42328ae00ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/302049d86e451f5f2fd96e1d7a51ae8228777d29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/32aa6572b2972cff3dc4ab39aadb430339a99576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/332c6113f0d6cf17bcde235bc455674e35f7b2c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/3e1cd032c231f7ca3005d6f31d993d2136aef532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/3f46c9d889a64e1f8de1ab5d704e900edc1d27f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/422b655f8d1c061981ff8f117ab434fc259172c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/464e03a9dab026901e1f6727f453dc92847e4f30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/4be5a7667c45c9c85c3342b9869c65655feff1b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/4faf32e40402fe19212cfc0ea0eaf2fc166464cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/4fd7db018e509a2424d77237486246c87e9b41e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/519c562ca220fbb8c425df4a5bfa43b9dfcb5b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/5273600d845352d15ecb5a73b7ff8364179ab190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/55dbf13b9e88b7c5dd9da3db543d4f00e13ad82d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/5b5b2f0e9226acc8af5307f84344c4a8f1c7a980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/5ce0559c275239cbeed958c3d9d13f119eaf9f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/5f055ffdd0138c6c4c5770294c5c8717844113a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/619b1226b81ebb61c6e6e6794334cde442c87be4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/623ffe46197bdfd1422de792bf55f7bfd4c2a216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/6a694f21749480ef4b93d17dcbe3a9589797cb84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/6d3eb6469d8562232ff429852d75a7552dcbd70e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/7155ac3ca88dbcd1bdb33a18c91d6b2a1e554fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/72d29a1160afe21e19cf88b91cda3dda2adaaf22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/74b0e172f59f2cb18d45cd9b495807dce7f110bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/76f8365caf68a2a1b0150a0747c6d98be55cb8c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/7b645de28b3f61ed37c57291efba1621bc0efc54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/7beec0e1c00ca95c3dfe480f735e6c274f983293.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/7f6b84ba66ef91b1287ae0ba382f575471353f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8162025a340119331350a68b01fbc0a5caea01a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/819264d9eab282f8ddf3d60df2a92b640a0b307e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/82aefd8af573356ca1e683756f070f4243eb8467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/863aa200db3a617619f7a141c1d4d88bf98a7ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8b0f38856b04fc1c0a5b79e1d4e02dee2e5f1e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8c9bf6ae0f943a19a8c364bf3b73f37d2311c676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8d59e82be494dba2b37d2c21a65d9385ff915280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8e64d84e86dde8af9dd9a5eebb4bd8deacff7f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8f2345e053931e6a991ecb7b239a769158e30f7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/9102cd8b58064e39f3d5d05ec99793b612fc964e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/94bc354730114d0cacdfc29abfe0630f6d47c4a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/9d85a1aca7f6e6d9d94ae4fdecd4be35c7576b49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/9eb09af4d4dd5631bce75e4e18ddf16bbbdd90f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/a50b71ff0c6f75fac5892eccd27a25c00241c82e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/a52515f115dedd34eb975d3021c0e3f19174b701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/a62cc9b00c47ffbf8a9b0087ef995c2c52bddcbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/aa7855e387193151ecf9cd385efe0fbb70d967db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/ab5d753a18c89229f5711532a520e6182e0dd36d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/b5a3938a7fa3b429021086bb7e65cd06b19be4e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/be3453ea753b86b47e38707d39f0413860885855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/bfb9b7e2abe470b4595c1edf01371d8063094d25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/bfc626e0c7649b321756bf564e151f9cdddba082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/c4d447c8ba7b5d05166b240bef77903f711775d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/c5e294f1633d7de169272a62258e6ddc04d847f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/c6fef54070c237488fa3ca91684ca8894d3bdad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/c75d4abb2ae956c6d1abfc5de547e6c42d869dee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/c84889d15de5ffdcc9aa6f158ed09ec3e0156d54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/caff15eb846a3278869cffef134adae57ca99341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/d2ca3523f923277658a83d1fa0e0287e8cb8cf2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/d2e63ced7164a73dcf3f716249b82c6b695f0572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/dafec31d08af3c6b91d79a7cc8d3b3c22d5a1aec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/dd902e0b6728bca859cc7b7d6d41c95b86b3280c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/e214460450cdf4c4acf4f5d8b065f1b647c78bd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/e7eb29532479a78538b25edd64ddc6a6146b42b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/eedea2a08c83a62f642251c564dd73d32ac5cbe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/efeb490f7858f3dc20da0b73b471d9762ea264d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/f18c2e565ff1037e9249720c3fdc0f05a7c26d97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/f7217c67e50e019bd0012ce46f9ffcd87392af44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/f92ccc5dc0bb7eef91dacf94a8ff5d771fbbc37b.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2018-22/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2018-22/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2018-22/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2018-26.yaml b/configs/cccc/dedupe-url/CC-MAIN-2018-26.yaml new file mode 100644 index 00000000..ba6d4bdd --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2018-26.yaml @@ -0,0 +1,661 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/01373d8f83d03a9092d7b9d1a28234058e023c3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/047bf4bc1b30a1028be790f8d8d86c5c020a8de3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/06508ee504ef9faf5919342db17f05ff4c061467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1106b6ca321c4c3f204c0110544e9526eb1ab0b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/7a76e9ad7fd4384a6e3f1446c4d1c579143ed238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/b5cfee863b4ae5e18c8fedb7602f07fd4144ffa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/ba74a77bc89efa8277cbb3a9b19c9e07c156feb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/cf13bae37745938c7ea8a35d073174c8ef783a1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/ebda20f10b4dd9a042f3f58f5f16213eb5889370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/f52727d379745eb68fa3b724b1c875c0354edcd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859817.15/warc/162152b5ebdb11f4640f22b1d56b8eab5a996832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859817.15/warc/2e37254220cff5d7fd135a146d4cb4c90de79ca2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859817.15/warc/5cfb2d2f6546c21306d9d70e0bd16f677b4d312a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859817.15/warc/79a78a6a4c9b9797797f3af9e357c1b37668b2a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859817.15/warc/c34baa5cad993cbdaa3080957f49f6472077f01b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/1d7e0b337837f58a6916535a43ae7c29457397c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/43f1983e99c15be78e091460f5691d9e7e551ff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/76a9a90efc929fe3193ffe9b89f2532bd775ca07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/9b2b43eebeace55ebe0bddf2e7e128958415b00b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/be728fa5e59e85d8d34a69c6113680f9701bbd95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/f239b94c6ff8f132a35af73741d48fe3545f98af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859923.59/warc/1b98a50fe4de991f342db4bda12681a6b7f3a162.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859923.59/warc/6d03c620eae1d1d267654cc7c5e04a2b71943068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859923.59/warc/b5d326cb963abf4ca7b4b776cd55557514c53ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859923.59/warc/b944d17214e6b798b9945675bebfbe6addfa670c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859923.59/warc/cd9937bb0e687dc400eb062d983a25daa5e47460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/0920badbfdd4aaae57c0121bedfd1b3fa598d660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/3256ce344e61fb9688eed6ba92d6e485bd1f5d85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/ba256609a9bd4b35d03904dd5db7d011f955efc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/d3ea6cc25a3d68a0655d997be97a2bb7f086c328.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/d491335692cb059bcae739876a99a5edeab69fda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/f11572574c037ef7233455e43e06cdf87ff1a83c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/023219af42f4f70f006a98a673eea6d591a59af7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/1a7948a29b01014d42afed38347bd1c4111f5319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/1c70f964403916bb27f03e56d589951dfa32a10c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/33fbb4470cf8f444eca5530afb9af06093d44dcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/6210eeb00e768e7def0d01f4dbd362c0bc5289db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/9a4e69cbb133362f4286399a9007f000c1a2d7b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.13/warc/1bab2e802242ade445c59aaaa1ff031d1ef3c896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.13/warc/44bdda06490293546a677209e9c31b95ec3c661a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.13/warc/55380395db565e822c9d1cee634ebba79cdae79b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.13/warc/5ef432ece6c3e39318eb23c4355510199102e034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.13/warc/e9c48cee2e9e6ad6f8726419cff8492fdabc57d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/0c93e279927a981fbb5a3ac1b4864b0e8ab6c02d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/3addacba83280e7d14b2a27e3ae5a514215726cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/466ed444024959d7b12aeaa8489627f2016c19a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/86387c623f546467e51bfbc032a5ebb63e4c284b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/f179218e3d3c0329ac6fe7804cd28fd3feb235fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/f34428f327b5d716eeb286f7c28833487b436f34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860557.7/warc/5adacef7d587eb79cfd7f052729d7d4bc6d38070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860557.7/warc/7afb1ca41a98a1c589cf802a9d28166ca89c2142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860557.7/warc/876639d235b0cc42cf082f6a75990a93d50a6d70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860557.7/warc/bc68729ed4764bbc775eb313c1a2bd3dfe91f2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860557.7/warc/d65a1aec5ac5663f7d7033e6167532e501d13802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/1816436a4e71ea2440e6bcf11a55ee6e75b68e5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/41eb8ecfe4adf62a2eb897cb9c223df02222fdb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/76f2fa9b40ef51777b1183353bab157fd6a04685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/cf8d216d8326e63902a9ec496ae46cba1920be53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/dec8aae01a2cb2417214731ed0965008f079782b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/e4533e1386108a96432e11952bea8cd2f40244d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/38e254e784f0ba03f8122f9f45518833de4c4f97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/47e7562207d8690ac42b8afc0fe8d00d873ebd5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/4bb92c40a2e19c115804c4c11650c380b84116d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/6eace2ec814fa6577228ea5161174fefe0b6c506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/f4e6b63be964540794983c2764508ffa86271da9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/fa56c7d6e817c352506aaa68ee5a374c534b4022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860776.63/warc/0d775b9dd38cd5f78b7a70290cabfdf70d8e1712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860776.63/warc/403d8420ad1fda71381d648a621e0b7efa387ad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860776.63/warc/48881282df6cdc7c1fc56ebf70188c5d2269abf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860776.63/warc/6c056e07366552678e962a5c733c7dbbf7849c43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860776.63/warc/9d5b4f7e8766aaf593600d7db5dde3ebabe91e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/1ba987903c7b6f9bc1f6826c437ae72d9ceb10ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/43a73a7a7e40d007ea65f6553f638130b9e414ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/58fe5335402dba1768091a4d3db786e50d8849e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/6fa0ec4cbf598ce7138461268fb3d5bb8b2acdd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/9709fb266702ae989e60f33e984d127c4d8a61f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/ccdbbe714bf90b3ccb483cc976ac23016489b761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861456.51/warc/3c0982c16ae70e21cff276fd774cba3ec5c671d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861456.51/warc/4eb3b76596add65c6f78b8f2ff4064183a09e9b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861456.51/warc/8ddcecd015d8e79fc8ea2a6e112630ed108455f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861456.51/warc/aefbbd04b52bd95c350a26d6e75b24e7b14a76a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861456.51/warc/caf88dba69c7f04ddbc411e737bd485aaa058eda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/073a02a6cdcb3e5bc20a4585ce10536860ce3d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/40b913d97f746df29bff904a49a85663ae3b6969.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/4305c5e177739545bb392066d2102ead6c856967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/54acecad70099457db78617802afc36eb575ff19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/69c4499a0c0128d0cd85d78088c4914cfa11a273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/9df7223be0986eb1254ca1d2cec17c6668601b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/109b93539c4a44f49c008fba279e4e14c98a0879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/2129bc767cb0ce3d1ec6c8d0d1094a8bee902ce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/5183f53c76cda5731c964ca2764d79e2c019b99d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/816af0e0d779c75cbac7911caf7663c82b3804e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/db40dca1ac353accf38930835dfca8b6856fc872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/e8205d895e49894308156d2c89a3c464f26772e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861899.65/warc/06c6402f27fd629ad29076485f89386c05620669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861899.65/warc/3ff2de261caa2ce298fb60ad1f8079abbef5f9d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861899.65/warc/503f54fd517bd772846873c9f3a6841cdf2111e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861899.65/warc/82cc3d71d06c43caa425a3dabfce51bb4a0cd4d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861899.65/warc/a756d40429075c9f303b916930ae7ef16a631966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/155da8721643f128fc49c6a38d98eb1006cff9b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/4288a02d7b0804beefeea53b182d9373c4b007d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/72cc59abc9845715f29136dc356ddb93b89d7bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/731daa8c03f94c5c9673df734beae6d994cca497.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/8dd42952c52789e2ad16171426c014dfc007964b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/cbb80935c6ed162eac6d54a68eef96214a96138b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861981.50/warc/21c849fae6d0cd1569d20730d8de1bca3c4a363e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861981.50/warc/4a9feffdb22c8aa2a362f7f7bd13cdaddc20dbd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861981.50/warc/7203185db6528049eea67987686da0a7623aea21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861981.50/warc/c6c4d731b42d8789c339d3d717f57662a26696a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861981.50/warc/f2ddc51e5543db9216aaf5438e863776d8903013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/081bd5e1e233681bb2bcb8c533ccf08e4f107055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/0df01172a57798ce307c68ad979aa527d35928a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/819fba1dc9c9eda62cb5a767ff0aa9c63e66aadc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/ad0447bad4f6b3af2002553908dedd3beebfa73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/c136f045a2c64e7a6d7df0e7987a228c92251657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/df4232ffe488c1f89bece245b6fea9cadce79415.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/398d1e649c4b88a40bad6463000ed5539b4c56a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/4db156faa4bebec0889d3f60311e88e3518ae5ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/67e49fd463cdd8dfd71d1a6b4000df7a57c133ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/7df0b1a38763046ef85c1f64c2bef78dafe30c0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/99ea82d42871d7873ac063dc7496557f05a2bd43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/d5c99bbb7059a9b1d0fa8eb8461655d44bca00b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863043.35/warc/0046cfce9b0fed21921f70a567535b463b24ee85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863043.35/warc/03fa9b729025c8c8898b2d3220b26f0e0fa6f0d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863043.35/warc/0a2be26d85a1b5f62d9285f94f88995738da286c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863043.35/warc/17002553012d30a9853e3361c3588908171dc4e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863043.35/warc/2caa93f44dc615d6dfd7845ce1aab7a4aa5e9f70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/2dd9b504802d1637b6f2acd5b2f786805dad9cc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/3eb7d68859262a1c8ad885ba8e19d6e187a16c10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/42cf82d71b99b9027a238be9185cfe24dc68c6ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/6aa2da9a6205f4a0e9e70d0c9ffb4623f4c63296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/9a4d4b4d37ad3a3e1d2a1bc6e737a47e1b9d82b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/de323fbaf3be5996d7b6d89b193d1600dccc7fcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863109.60/warc/00c2226b5a4e33585ab3cbca8846464c57d1c610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863109.60/warc/173335dca0dcb3b48ee71da040496cb3cfacfc23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863109.60/warc/924b9d0dbdc4a6fa90d317844d81d5d3ecee2379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863109.60/warc/c4d0e45b19da742e24cffcab11099cc2db4c2325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863109.60/warc/ff203c231b178777eea3ac085d8c52b40075c5cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/269164e0897ce5609474aa33ce5e07c9e54da895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/6d0685701728f67c8c3c685c6c78bdc524ddac61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/6d9f0e6ac5152ec0b8f3841a48ef05bdd4e188f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/aebb1bad0c772e8aad46523d867068308c5357ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/b4fe1edce9dedb98926295eafe133ff610bcd9b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/db87e455d67b2a1fe6bf249acfba6e7b4873b9bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/22aaeaae2a7475410a78ce84ba4dbb3c87c6dd83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/44f3015f73237a9292e53a55dc5f9020798226a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/465da1e7788c28772b8cdb30e64b23e2b1504fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/46b9b01edd9fe8c0c7467f8f28b6eb0f026be876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/947e51a309f00472974876b9980cfde3568e5993.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/a5502d89a1225c78441ffbd3380c3111dd3648b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863259.12/warc/5e594964903c22543731cb803af3f0e9714cfab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863259.12/warc/5eaf48c9ef6ba75efca8c06f3ee647443423c3a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863259.12/warc/712700dd59d5e07abb406c862ed23f1cf05477b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863259.12/warc/91db54d60d4741dbb0497f593d030ab1af1f4a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863259.12/warc/c4f3353fd867e491f1676170f32e6a27c2d505cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/2c5ba0be2ba4c9a18f65534d199f557c68944907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/3f39467795294e97516e6a9937154dded6202906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/4bee4c3e2e624a2809613fc6ac13591c837b952e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/66b3d2951b5cb34517dff274b8c3e9af23f0869b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/903ea7aa811a77f3143fde8e01190379dc8c4899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/cff07a6126a64630701026094bc7d2996b177796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863411.67/warc/46053f3e753700e7f3fc0a323b8ee4d3d36cf7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863411.67/warc/70cfb3769d01b515e14ef7524c94dc89fff8f576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863411.67/warc/9a802b58e6a360ff468b9ddaf1e9c1874e220aee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863411.67/warc/a45edd425b190be68eb0ddb268e1b4c6932fdbee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863411.67/warc/ff6e6cff627c46a1c01a59786fa66c5e4795563d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/655ab92978d6a67a69dd2b4b83d9796c0e2e1081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/b293b26336be4d785b3da4d5cc2778e2de570467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/c7d32219358e09baaf2bc3686d3d8f61651204b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/cc5374ce98a626b206e31be0ba0061347a5cb2af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/e51f757030b96cfa9395b664fbfc405dedf394ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/f1ebbfc23c1fc14d99a2ae3239301fd7f374ad3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/0d9532f1b998c30dacb3539a297751e40d29ccc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/4ffff5ae229a3878c0b9f3344b24f218ecee3eb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/697484d0fc3163a8c5d9a12bd23ee00199975637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/6ab855cbe1fe8b40f0a02d30819da89f2005b018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/85adeb5b205680c9dcbf66b17901563c86fd6891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/ff151b390d1b6b2fdc04a2cd53e25ad164a31de2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863516.21/warc/2bdca2b81f1ce462544f9faaf3bfe7b56c6d650a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863516.21/warc/5ca2055b3772ab11957eafeaf4c3b7b89c020abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863516.21/warc/5d2be1c3820c716cec52b6f6566e2b9fbb07014e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863516.21/warc/d189b7e0a03ed32d8e591998d450a1b7b90da0c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863516.21/warc/e3c0129dde933e4ab059b66f8ee82da41168c5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/240ec41879235118fa2a2a7e6f19e3a2ec096a61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/49515d29a0146bfcfb220ed011898c312a97c134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/4af426a3d2b4b74c5c8a431c7d7f433fdd541faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/86b465702d28a87a52fe769c49fb12cc9b1d4099.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/cc5864359ce4035b7333ba5d6a84687d16fe9a4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/da9b6ebf30b76184f79e94714f9a1ba82cfb4581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863519.49/warc/90baab6a04400a9384e8bf610cb60b104dbe31b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863519.49/warc/d10fcad4ffc431cd28729b2822e7e520b785a997.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863519.49/warc/e3cdb73f7b7315cde83f035df43336e4d1ca305f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863519.49/warc/e7aaddaa35587f10d025bbadcc647e95d0b7bf0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863519.49/warc/f33f3ea1f121cbafe848ebc1b108379f79e6fbaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/32887e8abb1fb83190472442e888187a681fcbae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/3cdeeece0997b5182bd70cf1050e7167edf2e2d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/5262befa54c39758655ea52884d3c3a4f2e293c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/68566bf58a565a3bbaf4195176983d0486d74a86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/899dac9e5fade4eaea6ddf4dce54e6b75653361e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/fbe3c3141c21dcc663ea400ef0d523a985b6323b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/743e0250596e42069d3b57a5301b0931209f9755.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/9d830945debd7232bc0b54b615c7ea6722cbb854.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/a0ec920e5211a3daa4eacc7ff65e256dbdda78f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/b91b0dceafcef1e362c8589c96daa34288a4c91f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/ded0f820791cd5fdefc3a11209563a83fa145b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/efaadc2fe227a0b150d8396b97a36e5a51796624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863834.46/warc/1ef28e5ecf065e611925e5ccb7de4fae8dc7cafa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863834.46/warc/1f29dc02895d6c6dd963501961d7412d4c5512dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863834.46/warc/7e1d9ee8a3916ed2210bb3001a54f10f69c4fc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863834.46/warc/a4c5b87676a0c0384426dc5cbbf2c19cd27d38c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863834.46/warc/d84794a7264010afbcc2bfa2dab9fb81bad83ad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/123605d186b03e78f5ec9d5042c53560e3af6808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/56d2bc16d1e55e860f4e9bd8e5c513d246429836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/6326ef49ccb70393a5d2ff2bb5f0b7cdd868c84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/da96aa7f5229363cf5de8daba84306acee467274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/e5097bfd65b55d7e9a05a7735e3308f4235d0b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/f417fe51d3f03cd7bf409829b9fcab32d910d3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863939.76/warc/7c088f46a12e94440de7a22b64fe3de991abbbc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863939.76/warc/92631fd1c9c06f6e6bb3e521f71c25fb04504bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863939.76/warc/92f87b3867960eecd4d2bea3d14dd19b08256165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863939.76/warc/a9c5821e233e839af082745e00b610ba91c9af4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863939.76/warc/baae38147b4729193a8fed25af5e5610653cb74f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/031d2ac53167ca98c9b1da94fa3c94aeb5a1ed57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/17e443df835b200335fe2e45e85576731ac99d90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/1a1bd58d038bfc22df5b7fcbf9c03fefbdeacd49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/79dc77beeabb4b19d274b54b8a22240c2ad266db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/b7768ed44b1be2dd6250fd74704a5b946d8aed8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/bf101d8054ca59bb8089a231e5a28a58e9357d4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/00ee5f4b99a6428a125c17511b58d18b076cf29b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/391f8506d323f9543e7a0354ccfe74e7b1d5d8ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/89060b504803a9ef337f67b1bd7614ad069f16d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/8c1c1914cee7f76c191ff65f234c8b69b0a17e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/cab6b5ad85555d79f7bd031c1479c42b9268ee84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/fa2a8988e1de97a6b3770d536c5843112ef26416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864022.18/warc/19f52b7440f7b153c291104ef98b2021170dab65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864022.18/warc/1cd8ecf3705353806ef59c2630a32d33ae87c299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864022.18/warc/64b31e560483547a84f31aaeeda3872d02782aec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864022.18/warc/a42ca2bc26215944617cbb2edbcca5145de12424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864022.18/warc/b726dccb6b0b7a527145127fe35576c95d7ff853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/791f622e623d3c8b317cac56b3f0817618143c64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/9bf1a7e06647834653c0bf158533a76cdb1a15d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/a0e8070c38ae0195c46a12c2edac457d48a0c0f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/abe2d9f7e339d88604bee2b5b4e371ffe6de9555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/d87f7726fef55da853a2f92a2094885fcd4f4878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/dddbf24bdc9f9023fc32659217b11eaf6830e9d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864110.40/warc/08d9e9cdb1d88999664d4d9b50c86e78d754f0a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864110.40/warc/9457fceaf9ae92f989456a69c0729ccc39acd3d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864110.40/warc/d622cd8c0971f102baa39e34d3d15055ee598296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864110.40/warc/e444a39d0f531b510e4366eb4662e76d406b592e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864110.40/warc/e5c9aac87d32fd52102c2efe0f6293debfda4b30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/19d4ea3bc8d813cfd3700b44bf9d152b73131f4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/58c9604e04423f72690914110072275fc69a945d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/8785574ef753bfb9c1fcabaa1e9afd3945d00ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/a187ebe7de560643200249c46e6cd2c80a025644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/e8171bc8d68717618c7bc5bfbbb0a5599cea1f8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/ee31b5c21ac35caa21b8275c3610b4d3b8fe8849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/2df320e852207472e61d3aaa431baa4632421de7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/8435254ac4d23cc11f135477fab843557aea5aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/98a6fbc735f39828ab5b27bd6a95f84964ab4a42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/b598dc3255791b2b25103168d1a05b79e2fb0739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/f5d23e6bbfaa4245487b818c085f454bfc11edf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/fc0395bf9308b1a3ac44ba3c0638d9fbfd3ae18d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864172.45/warc/341f6fed89216eb78e31549023bbb449ca8a801e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864172.45/warc/6a667946e18e2e87207995bfa55f87a17244aeda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864172.45/warc/c3b0b7d182b782d7ee16e54beef092d9accc788d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864172.45/warc/e4c7928741b3e37abc62ef29127685404ad910f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864172.45/warc/f7c78af331d74f371976079dc3bb122abc087e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/0a6b790e518cf641a79578bc5dfa25a1cd3c0f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/13f3e0008dcab3e6dad60fdae85d344bc4565d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/1895d9e53c42d362562758757097afdb5adeda62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/2d4a03e10960c69014612dbb0b7e58eefe73abe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/a394cc38aabd6d8001d6c979e7d70bee5d40faf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/fd0705a2180af2384e861f2c6e411891ffe97ea5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864256.26/warc/16e5d28a466fe251cbeac8958137fdd0e164e2f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864256.26/warc/73c76919543b9067ca0f757882351d538ac4b1d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864256.26/warc/7bba56d68ed697a1f032409137f6c93675009cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864256.26/warc/7f23d458c0c7d4747b9146b1c770ac8679edb5f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864256.26/warc/e50ca26d875bae8a302cb654b1a60f7777c786ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/0e1d494ac403b5b67d7c395e8f6c994c198d4b91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/1c7fd7e9a20f1bff4925df96187cfff0e8f2038d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/24db6c6384c85610c9ffab44fa6460fc5888401e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/79581abe63602b616843c4674545aa2022ab655b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/aee0609157b0e7c5324f6d22e08dd25f74e9ae33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/e2dfd7ad6e289f687cf191bd75b91e3dca35157e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/966a28f5e14180671e1b99fb5159bac38f31271c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/b1b2272ed024fecfeecb9f2c2206fa65f250a999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/b65babb7fa1e99ac15b76ced46c59f766aa1555f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/da47c49d184ab34299b5fa696e2913e9c73d0716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/e6398200738c89469ae90556be5e70491e091f60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/fb2d7c9e7d7cba981691470c8c1bfa208931fd68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864303.32/warc/0ae7f833c7ea82046aa1af62573e99fc577be13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864303.32/warc/45049e26cf0452890ac7d063dd71e87a1072de23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864303.32/warc/bc121be8793f1f7dddf651c6da0db1513928c46a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864303.32/warc/bc9af692b1294c45c63c9e0d5b58af66c8a29e6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864303.32/warc/e1836e041344b36198e19fd1366706a9b4c079da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/2c3aa9a73066172e7b870d8bab3cab189cc10c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/5d8b7dde7dc895eb80e54c8205dc3b0da73a76fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/86d5cfa93d9becae6afc1bb6b1e27e07d61c3078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/884362fcce18b988edba9051b9409620798ebb11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/c9c38fa08a4c398c6f6e34d832035748e59295a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/ffcb1b958f7c29ab907854cb1164046eded37abc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864343.37/warc/24e077be492ca713f46cea18d8d4c821c2a68e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864343.37/warc/4a09b0480d3850c8153f5fcd8f6b9803c11d0678.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864343.37/warc/854e15bab6bd3db006c1eff4a0cbb41eec743f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864343.37/warc/c39f0a9401f114a014066391f8fc150903b1fb12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864343.37/warc/c3c1136d2d0b4b6b1cc437ee09ca9d324cd8f66d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/0155e7492d20bb70ee95101742e397e9b8c08675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/1849d6b2227199f693fcdded715a4e7ec6919935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/32154be98ab6b7030fbd7e1ffe8803565128bf72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/5de370f6a1ff909aeb4301c3277821c3b92e87aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/92c2e55f2130a4c7055aa9b54dd20e77bc490100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/c308c82b911cfdedfeab1233354ab5b979b50a21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/055c0a19742cc528f4b048b2d659bfae5ba6e1bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/2204d2f1e6a05a3cf1d0a714a0889112f76b741e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/66b1856322bcb6d2489d631d1bd6170481a2f0c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/d62bd4d0baca2c57b1d207ef042908620aceab42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/e15d0b439f085c9e7ee193c64cd3c99a5a9d35a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/ed37ba0be55380b9cff8aa9397072f776cfe8ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864387.54/warc/25e7b444442324fa015a396eabe9b83f2b12bffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864387.54/warc/2e60dde8df76c9c44b7a3ddea26077dca286f868.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864387.54/warc/439f59948c33aaa4308e1c464818aecf85ae22c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864387.54/warc/76379e97f05d4c6de9e38864773527c1fbb888a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864387.54/warc/85d47310ba53af31770ee8ce2dda275631f21412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/2792d84b83153282894b7565bb004dc8cf2c3736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/6170cbb471b24a6c3eb28c6337c59eecc74f78c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/84d791329e41fd9f360fe7bce6cef2e80f41ac20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/973dedb48f54872c8d92d1ff969408237de4ff16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/a5106622fd1c373a4142ece681b20eadaa4d4e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/e66e5121d01293eca6adfbadec39f1356c5478c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864482.90/warc/4c902416774b608ddd7c0f02dcae81071854948f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864482.90/warc/b4418d542bc4d309446bb60ef3459b994acb0a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864482.90/warc/eba3657e433d7c088947a0be544211ee473a492c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864482.90/warc/ebe35f178ec49bbd5e2a4aeb4bb6459a6a138c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864482.90/warc/f314c3972c51be6b52f8d2ab3d3bdabfe6cf39b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/0035963c416f38ebb5d538ff1134c8fd9b6ff9f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/2bc48392cd023d346758d30ec303d3d9e718e350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/8543ec170a53e614ec6648a10c7f13d29043fe04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/cb46c7f0b432d98c69ac4213d7249f3538f2d942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/cec7c5800340d5192dce96f4aa43956e630825ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/d7f02acae23a6a3b015bf583e80c23bf28e1bb46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/5a72d982d6fade707955be930802228e02d746c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/6c95cca594a7eadd7fd3338139cac67231441daa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/8d11b3b2bc47fc6301809c875df1d87eebb46e26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/9d30c27fe3ee73c1950fefbe59f34871edf89beb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/b347bccc5856d962c6d5e496756e2ebdc6442aba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/d7f917556c96ced9553aa0f9c12aa3e723da1f2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864776.82/warc/12ad0b3b1c7e2f3c0be2778536fd4e9dea92e76a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864776.82/warc/14416decad2dd6a6b7ea11f13785311c4dc2e1df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864776.82/warc/150c0c0f822a1ca922d71daac3dea195b824757c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864776.82/warc/80bda9ad2b7352750423b252bf3d371c4e709240.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864776.82/warc/ac46ee4f10edcc551aa4376d4afdc1d49d859ad8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/7cdb9e3f948b205c342f301416e420b6cdec08ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/9b1f1947cbd5d25a2f64f386b580c59f90caaea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/a53b33d53873b704ed7e2eb68a2ee02866e237ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/b90eec93dbcd6a65eae3485b5ae78647ed53d021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/ed813240ece2fa4f9005c30a9f6b8445ae8b0ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/fe97f8ae275ceb5a12e6346cffd82cf2037b3956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864822.44/warc/02986328f8f4e5d407018c5c688831a7ad336dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864822.44/warc/2086859bf3dd3be14d36457b096970cb1b638288.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864822.44/warc/4f41fe5d5a2863b50ccfe4d0159c1ddeaca8323c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864822.44/warc/a3fd527b5fb0adcf69d1dd76058f7884a3d24335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864822.44/warc/e8023900dd1efaf78b20ff5c3b71074f9bce0f5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/0c41405694ad993c207729b42821096c5fd20ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/777ea296853abb461deb123ca930efbe83d38780.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/a02310c8a63e8eee6a7c4c050966fa9600b0c490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/b160283f9aacf2002abf9e8af443568fc0ce187d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/c616bfe0cb85baad5bbeee3576766f94c494cafa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/e680574c859130a1fab4b67d907743b1bce96164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/1ae25e84b196aa74a645643ee7df3e73db5c5b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/22f0513786d55bbcc3fa3f49c115677f03861cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/437570dcd52cb625e1e2ed6d601cae50c751708f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/c182debf35c1e774a0a8baec23b43e82e9e7a67b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/d745ef112736c07554c39a986e6664fd9ce263aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/f0f5d8db7aa0deb7d836b85d954f5c796ff3182d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864940.31/warc/475fb181d9143791fca90a89865e22857763f33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864940.31/warc/573744af410ecd7d7519d99fef784dcfaeb66cc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864940.31/warc/7c1e5015fbea49fad99775bb2944c7a8854ded2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864940.31/warc/999f6ac9e786f21597dcac97c1c82b3d9041b6c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864940.31/warc/a36108a3c5ccdbcf0381154b4e9a94ef6171da08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/000b8e3bc9f14e8c7f6d4624da3d63469f841a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/0f92a5c2f5869753dece58bd2f9902ceada9d7c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/842cbf14f032ac13d8db8c5442be087b4a3efc69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/98ddfd95bd69ccc24c325d616efe1166c252cc5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/bb46cb69a7d637a460ff10a667c56caab6d42401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/be9d01c335449f5a649a9c1d8207efffba875e7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864953.36/warc/753221e9e304d12fb4e9341c1a2d0323e57865e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864953.36/warc/84ba0cd492a91387c20f8a27a0b68d60fcd57ca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864953.36/warc/b5da281cdbd2f992c85b5ba310dbce50069939cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864953.36/warc/bb4c07d564f6c42b843540bbdec4370cb94a13f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864953.36/warc/ec3c9c7ee03a37806dcffadafe96cbf31f3983fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/64243888f23f08c249dd7a023a1b3ee04fec80b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/8836f4e82843f8d05415c9d490a093264b4d0622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/8870bc6e7b21b9a416c17dc3b7ae655fe4f2b110.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/c1be6c330f92c57f78ef296b241c00b28a8db7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/ddaa6e44049c612608cf62e9f8d5472e6ee2600f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/f9766df2c1c7d7c0d91f01f09a5700821c8d4432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/2b12e39fb3539611da2d76c2eb29433e87af2b4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/37cc06375005f32b0c41102e7a07161d0dcee6bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/6fcd8821bbea28168223aa0d71b6b96acffd0e5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/7282c41267549b2e68b2736bca15eeee8da9208c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/9cc63a4b758aad8040df8ff1fab1d376aed3d4cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/f89eddcca79e1019b817182d78a68e613223cb8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865081.23/warc/095ff9ffb41dc6ded47c1655fa241a100c36055f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865081.23/warc/6fc81c322fbbba978cee35f1bd845ab792377065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865081.23/warc/c41d39c8eccda2aef39e11669eba1767a6112b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865081.23/warc/d90a21a90ed545859d4c0f27f6496aba3ed17df8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865081.23/warc/dab0059497b5e1372cd90ea370640cafa7fef838.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/1d023aac08f8537534b7ba956ff3dde13a47d661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/21520de2968b50cc30bb0dbe86251b11e81e8ffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/248313a2c2b48595aaba6d4aedfa20d56d6075c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/3557ab87efe4d6a90ad8bee9c4d9a7d49c2d89ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/52584719cb5a6c01a8f06ee3fecd7969c0f3f1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/cc434b5f2111cc68b08f72094147b7dace40d4a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865145.53/warc/4a8143ea8a73765804e3a4938254de14ab6eccd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865145.53/warc/685b9a32083faeb187c922f88654c1de8ac4cdcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865145.53/warc/90e590226c03fa0133790597a067a0a75a97ad81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865145.53/warc/e221c1997286cd598d9c1575e7cfafbff5c10821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865145.53/warc/ffee9f866d7ef42478d8e800d721aa5605a5c40f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/1a5f601def63f3e7f361aa7add64ac4fc1f2f242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/42f0df2ea828fbc9f4b28b8cc8bd498a59753a20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/6cfa2bbf82fb3b435313ea353c068848d760d808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/6ed53bf7e13872d556045c177421e4353fd8768b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/9e27c698f0118b64edbee7fa520537df92fde46d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/b53ed5d19efc9f264a568b87f3e560bd97243d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/73235ad9f176ade6db3cf7a288e61c5421680aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/781e51410ac572913775c0fba208b57c6699bbb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/9dd4c851734ae23607d0146b3e63af2dfcec8fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/ac50b95e0960f5a6c8a59c9c797d1bb3324499a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/b4877784e93b168ab6e83fa13a728e1ca31cf64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/bd7080315f9e93f852a447783c03469f1f1bb1c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865438.16/warc/03faecba6011b8c23f50f323937219c7b7ce15e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865438.16/warc/5abba2bab7d9a21e39704c8798b242a6ad50b899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865438.16/warc/5ecb9a8c9732951d890dc04e4b20b1fd6776e4cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865438.16/warc/9b4b21825aa54b099134d452abc31f0283f32080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865438.16/warc/f7d0235a37c4a257fce5129978c1667f16d35765.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/1718f718516dde4824eac36446c3efff2d83f1a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/5b277a6d213fe14eb258a55f0d73957981005756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/7ec22ded47e6e40bb825877ff07afa05559e3de0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/ae3fc9497598c5406893e468429d0346d357e74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/bca77541deba12c34cb2f96b824264a341aaf749.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/ca37801be2702b4b5bef374e43ee6b66e6aaf375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866191.78/warc/47a7012845c8944f5c623cc82ae2dc04460bab68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866191.78/warc/7f2839dd072ac85adc9338321e53b4a5dfb266de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866191.78/warc/871d96e6a1add6cf9b5c479c63eb0a2dc6527337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866191.78/warc/87a7cf56a662d565ced9039aa3e78b4c4d654a23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866191.78/warc/a127f1e32bcc35ca20de747cf59a50dd804b6e54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/1202260296360b4ecf6a68c03667937b18e7d107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/49d352151421564284703c773bc4ee7488f8edd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/838c2adfff8e6f9244f6f19e24df0380f6046659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/c05821e3ddeec918bbf7fa5cf24e85d227d398c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/c8c28a41106c8b19d0529847e4b85627efb92329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/f524ad8a23f71a2d664089d6e0812428374e87df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/17d47391ad1d5e230d3c24509e0f90133c5669d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/4b1c2e2b7e6c2f060f48d372490072e1f6d7b668.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/650a6e518b2ebaad3e20b1099dc80de9babd2eea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/9107ba2d573fc15e9c2dbe6830e7cdec13dfd292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/c4ea9cc3c1b27d207bfb6a9a3e33406b8d358a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/ee242f54be6096bbe3faff9fd64120b972c6843e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866926.96/warc/2001549770dfd037e71e21e68fb961488995224f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866926.96/warc/4b6d7107cf52983efb68495e211348acaee5fa24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866926.96/warc/84da8bda3d7cc2e632b1e9dcd3ab87b4757047ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866926.96/warc/e95f8ec5b92363c7ed445ebf56228927805fca3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866926.96/warc/f9d5d13c90f3c052e8e34b13404754c6acf416ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/3ce9e15ee9cb4ba0fb599acc9bcc0413a4d9c256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/4abd72d7085957c3da01c8d31a2d034106b48b69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/61a3ae758e46e089b3fdaf8f7d819d18913faf32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/72f16ac1e9d5891972194f3f0ead318f540ff413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/bbed7d9f9215396db4bf5c0519bfb7817ad05fd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/c6a739b8ea6c301fa52186fc62d917b9815bbed2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866937.79/warc/0238d0fda136ed6b6688ca84b386ce580d151de2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866937.79/warc/1d9f84bd54b4f01e2a5ef28e986d735718fbac73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866937.79/warc/bba9b05d8ea503d4d2ee30138d6b4b15d6d36551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866937.79/warc/deb67f342c64909a22f6276446e9e2a78f54c88d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866937.79/warc/fa428caac542ca512b9966d88d4a545a3a668e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/323663d39905babe3aa4558f869b838fcbf18947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/565713e2bcac1b96a56c4923d699ed0f9d7faedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/903ad1c0d26e4145036968dcbee6732db3ed6d26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/9f474db59d1640ac51b16c968b38a6ae6689f407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/dbe47976919d2c68823d1e2b9ad4e795fa9e91c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/f2494bd29e5246a6f379c90fca5dc84403b2a127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/1ecff34340c1350ff9c4bc6ae766e67137ed3762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/5d7218cbc1aa8848592fd513445d10495cb45933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/5eb86f0844bf9359320091bd2b14e2268053cc86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/6b2ef829146ec80abd10e9d91a11f297eb9d68b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/7982c0ecf52915e93697020454b3dc1fba0a19e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/d53ddccf42c9c838270dbf9e6d227aa1d017dff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867050.73/warc/3a901f2486106b26ef35f15b8f29c23fa3f14140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867050.73/warc/41a1a3a8c98ab02a53641d65958cb5862e041d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867050.73/warc/4ad9de50468240f7c9ffbf829603bff4335fdeaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867050.73/warc/5f82ff8f09d8e67cdff2ded2b7363cfb057b27c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867050.73/warc/ae5e86c47c40433c21f990ddd81f4d030eccf776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/7546cff4cf3b1901354fbdfd6ec227920fee14c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/86dcaa45310ff3a65d5b467e80c30b480b69e642.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/a29fe15d470725da52b2a59769527f20afe84fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/e82ed9cb3f941f23a108972ffd400b7291ec33ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/f4873c682e649df3e8105efda9338b853e1b05fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/fd3662bff16892933aa6fac5bc8a5402c1b68431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867095.70/warc/3ed3f6ea84011c03bc083cfefc20cec1a232abf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867095.70/warc/40267f9b633e3f87fe044e1581ff50d94670fd9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867095.70/warc/b8b5683a541f4226a3ca2534867f903a5cf64673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867095.70/warc/ba96cabf0a318d06922902a2ec876c07fe22fc8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867095.70/warc/f12cf3d0250dea2c3f677a6b5f03a1f2f1fa455d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/0d06a07996745adba28049cdb913eaa9738f60f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/3f4dbb7188c811a19cdc5bfa5ec33cf2f771ae4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/4c168166ab8ab17d9cfbe07cc0da74e203cc756d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/b8afd7a033cef7880a5e5da745c2c371a0d16133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/bccfe73e219af1871aa4c6fbf5c8937aeb113e5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/f15c9ffc980fbd4677498abde431b1ce648a44b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/03fb619c0ae979f1ea2c5a39669070255637b538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/18f4e6a945b7e65ca4f0ca5c0238bfa53c5b1c24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/70dc186b3ab3fd3d7da2825821fe189220520495.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/ba7e273537fb32ae424c6711a0d8d1cc986b6b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/cbd1aa3d14d6d640568ad8af192061ceb5a93d6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/e7ee1b64ca715b78a2ab87a0cacd0ae07441f694.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867424.77/warc/158316e7ad65d20bedf1660f75e559dab1dd5d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867424.77/warc/6553471ec511c5237fa2634a4a002e623d8c35ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867424.77/warc/6cb282a48ddb33c209804e005cc6735f410340cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867424.77/warc/9cdc7769fd9093a31cd51f0c8622d6e1e64d0c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867424.77/warc/e8137dc0b6dacb61dca819b5f32d4f92a89943da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/09b6482c5c2cb68fecb2ba7ece590277b89eb002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/104af4a0b53a016893b4ae665dfc78ab27c1ef57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/7df2050ea5802dd5642401720466eb0b0f6a94ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/9d7f70c44e3fcf049db4f387a8cad69839957d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/b8cb3b12d4691047659986a87a915466b20a8129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/dac592c8c88f4b7e76520fd9265f58fe98f43dd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867579.80/warc/48bdd43e2eb6bb39a209f473c401ec366a88b66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867579.80/warc/6d3065ff98effdb452bf36c52a521c5cf68299f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867579.80/warc/6e84b51c79c1003b2693c2630452a41a547fce5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867579.80/warc/72350ee2520c1343821df61964ebf1fb0d3f733a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867579.80/warc/7f0d523b280a65998944c878d5718250db583652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/30d406f06aa64389ce41d89beefbf070255a89c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/3df910060974fff1c07d5b6d5e4bbf424c18ee4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/805d03bb839e0fdd99b66dfbfd54d9f576e18818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/b03b8568b7a75a2d6438eb13a85896a9b1d2bbf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/b6fb18b9d8c2b67014f6ff37975827f0a5299f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/cbabc6942001a1ac541717f1a78f21c863f54ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/461141ef28c903748790c005f148f2f7d9ade84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/5845630e00717316a7c6ba43d216425ca362b8a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/83b04d8402455c9ffafa7316c58b6eff878d7ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/8c7e3034c5f1bc8a35e30f5fb3b70fd46f3c7b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/bbd44d6b4508c6f78056719205ae1fd8ef0e6dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/cffc0993371e0faeed7449f77404a17a96232122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867885.75/warc/096d65813bd39bde295998beb5b5b5730d74bc77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867885.75/warc/32b32626d037898d773cd512fc6697e71bd132a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867885.75/warc/34b4dc63a62833d627238694adc61b088c6e04f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867885.75/warc/82ac109b6051fb12b5907687eceae57b22b263e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867885.75/warc/dc3dfbda2a09cb99ea29a44ce6c103178abdef06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/2a93ae0316dc1a7bbf03e48d73ed47ee53efe59d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/3097fd637aa0d777ce1ec682725822cac32c7db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/37bfdbe3b06c88ae8071bdcd6a07426695cc3aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/84e69eb9745704c0a38cb4454ec8050466c901e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/cf438466567059c18572151231f414711dd50806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/eaec6759cd58cd4b1a11a471cb2dd9e16948bf04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868237.89/warc/2db416fd8c706df988d34e5c623187cd810760ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868237.89/warc/47fab17cea952b8ecf797bdbdc8cd2bc5521b577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868237.89/warc/5a534b634906559109823a10ea3b6cdefb9a23dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868237.89/warc/6beae0ffeda81261ddd2ae497921ad5c2820480d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868237.89/warc/bfbd854210e8d59ad383ae2f9fa2cc2a3f999ee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/24a919cd2e06c68363a9faa5fd0fac1402499b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/42dde5832b86a1ce2567260f9d179c91560f6ff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/44c8b09d1e7d8948111bb17789145271c29b5b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/499553269e0fa1b7ea79167495bbaeba3128d74c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/a1398ef4fdb48369a61893ef976c1632ed537781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/c50c6db63c194b275c23ed2c00f845fd84dcbd47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/156b42efdf066676bb1377a311966f3754bd1d4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/15ee433f55a19578a522b738173a9d86f5a28176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/16d286146725835482eae327c35fcbe19c5a9b0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/16e16fddec138d6fb9295eebbadb6b55ac08ff8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/174ebb392f77ddc2f82819a78fea68269144bdd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/20d6b09edca219f92f263d53c2fae59811f530a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/21d244c378cec2982ab6704e27bdfda172ebeda4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/2490ba5ced70b459d9f92c3d051602a55ab8d46f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/26500ee7b1031c56f71a635d501faaafca01407a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/27c5ee40f02e21baffa8cf340ec316bf57334d61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/27edc2685d0f7dbd08bf3d940a9175dd7a443c37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/2a006802f87422e36a6fd8287f90768eafab89e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/37040527cc0a06591961034883943a3eba4a9cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/3b72a1bbfad89bbe905eabf66b70dc81e96c42a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/40acada21ad018bb1396df5e4bc97b74cb194097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/40f2ffae3a511da182da0cfcd6ce16181d75f5df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/4143bfb2b942abc39055d4b4f6fe797010aaa8fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/42507ed1639a5a14a899af7227ba73c957c3923d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/49c63b5c18b987179c0b483c5c018e7d03f78402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/4e12f7b72957f4768a4c6b3e8af227487df81d72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/4e4df696eb18d1fda00a0aae35027f8615deb150.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/52d2571fa02ed850361312ea7af32c04ddbcf796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/562014ad2c8d248e4704b0b29395fb9b51d7ffd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/588f518e2395aca5679df1e9cd7612555a5326cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/59b7bd63e4883cf2946615a98993a45c5c079fb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/5cb4fabcd65c98923257b187cb865106f2caee99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/5eaf4b75671dadde8118369717835192f7de2f26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/60b2a2d31754ffd7dc771aa8f4d169e003e8de5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/63b80ca7b367cf8245430f817fcf872796b854fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/67febb409b36a5a457192a8f5045f3b8caaaeba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/697be789339bdee78f992c113268616317aa628d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/6a24ce8e74c2b6d42a76cb585744264a8c3c32ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/75ccd87c1515716c4091d71529e1f81be198eae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/763036f1ea0bb552a62289ca33b8383ccf8a246b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/76b094d857a5a822c0b788a23c03043709cfec07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/80d70ac840ed1d7480acbfcc45dde5dca8b875a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/86256a4f89e5927629932b20c6cdb212b66710ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/877fe730091da8b9c06568604ea2cad04f29d85e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/8b4b7899e2cabf7f61237fe4618f0caf302a691f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/8b89ffefbe31df8780d9e8dfe2fddde95b865729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/8e9521c105029c7174703890904922b4225ce285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/8f9212fb0f3d1151c791013498c5ee13daf0f158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/94e2483bff4c76e566e99d10860fe3dda7c4d742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/97eab51950d38e35ce7c098a19952e0aa2ee5c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/995867d80d73d9971a5c8ad42f44990de36c0ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/995f2970f54397510c85f9bcc21960e06433b2c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/9972fc7139643b5f46d9bff3a99ebadf1adb13df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/9c04e9b7d6b129b724d9d4f75fecc44e09d3dfa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/9c27bcc96568d62a60fce18760632cfff253d478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/a0030e487c9d84bc3d37d99ad5d6d03da90fab67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/a8688608818db4517f4d32fa284f30838079279c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/aad964be83176589389e899b97deba7b72c8683b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/acd9391c2372b9c8c2d2f304e25de29c37a04952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ae02592b68f5bb631745422a8cbba9198b6f65e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ae63d5eeef39ddf6a7335181e00fe5a0ba8aadc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/b768820b19c71f4c08accf0d69577cc79f6d7f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ba717447a8a6003d241c73d7e88d4bfc47e3640e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/c4d6b5d9e8bc6d6ffbdb709f049e4c2ea01ce5cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/c60959d6799f9f11348eec1ddbe7d35a01b248c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/c8de42f641e73b223d512bca8414ad7a94c67d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/cdadc9a7ca6eb7d535c71cab328e1134a2c7a562.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/cdda870787dfc73812b72d9facb8c73afbd489a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ce15252dd1d3411863c8ccfd05ff4271a2831750.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/db842d4bf5cace03275be13f840587b918c04de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ddd67bd631194949bac4f594c2ab10660b1e308e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/de8016cd0adb8253a6e9f246a580f472710aee5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/e18619945d4c3b3ef5cf008c88e41c7db311842e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/e1c745f7b102c60c53366a50da826c6e6e136b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/e6339a94dc8af0fdbb8de6fae4e8aa891754ab27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/e9a1b81ea2c9e45c202b05f59f513306e33a06a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ec83055d143f3b2296d8fb3b27ad70e6abbe7659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/f7b19284e0529826dbb2dd79334a43577e8583dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/f8f6c673ad6a542cde62a6efcf091926aa303c3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/f8fca67b04e1e47a4efe94f81cbf3480f4c837b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/fa3c62a0875223ca310a5368971c4e227031f4c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/fa442e58cb2ded3f25ebab548ec215871f0f1046.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2018-26/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2018-26/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2018-26/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2018-30.yaml b/configs/cccc/dedupe-url/CC-MAIN-2018-30.yaml new file mode 100644 index 00000000..2ee61648 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2018-30.yaml @@ -0,0 +1,661 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/03eb0847729817afec3dc7dc0bb7937d55979167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/064900654ef0eeec943f8601c0b261e3517be228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/0825e5867f16994f19182586039ab98e2590b576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/097ae82800dfba8805058c118511824f78f1eaca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/09d45a7c78e1fc228d80d685319c24d65c995762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/0fa5594c4293f0154af250794580bbceffc94b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1125d3039f2604e41bcb12635ada71fefd9849f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1162369243fc4dd0f09f18781941baa4aa6e4a06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/00a30b91e3af498762c5fc967f49471483263954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/155c9f961d5a50299a501438344d058d15dcfdc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/605feb9123e482332bb4961112b0c89618949eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/8279e125308d5b49b11f53bff880ae7c400e0c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/c7d7ed9bebb0708267692086474b17bc4f80670a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/ee5b832778adbefd31938fc24c28b52031f235cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588972.37/warc/09585c4e642deb318018b7e5f89e0ac339c0d96c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588972.37/warc/3dbf524ab7ba931441eb5028c8ad1a21d8068343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588972.37/warc/a2f73ea2edf8678d203604f152ba895a4941d637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588972.37/warc/d276b11f2cedbeb67a8bfe0bd987480096e4f6b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588972.37/warc/ee931076571f51eeb58fe67998a704a0ef64d297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/0772b3a56f442e376b485d73549e83dc0047a36e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/4ce3eddc5a4ad4ec7991c7b58cbc0d6d5a3dc0dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/648b9563119819293f9c6ec3a7b976aae53c5e19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/93f6e12640554a8a19589687d096b3dbaac3e525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/95bbea578b5ef35624e58c96477b2278c42bab4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/c385be93318e5352f18e6d076ed3502b7c93f66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589029.26/warc/0f13ffea70d91173b99c7350b503f6ca355396c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589029.26/warc/460e0bb4eff96f6c1ed99c7fad9a8a75a1bf68f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589029.26/warc/db721a6ca2a71fc76c310d81170a273b7fc7ff4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589029.26/warc/eef542e7b7769b5d0219c10c9998832993ea83b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589029.26/warc/f70c485612f948b4b4e278a95fd844cf79196440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/09a897f66d79035a95f6d482a1dc1f21f46a9e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/2df94e197dd34f4d38ff08aee5130e2171310763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/87f93236fef9b7099af70dedd25d6b6641700d38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/9f7a38ae05d4e8d6dad883b6228a7ab55c6a1411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/b55ec56a6787b3ae408e56f3f1a12eb412a26ef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/f36711e07a455568499c6becdda307db917eee3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/0a42fa18e8a3685096743b8f1b9d15570845ba48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/121117fa4aff708d2d7a52d7649d6a741d95109d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/1adf5952088f03e04071542df5cc7e5a8cc95108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/59b24bbf92a12f5f7993976a1c938744a5e1d2db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/6eb807090ffae5218cf4a22c32705c225455de80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/9c8c0508a55e0269931c9d3dc4e9f39bd26e11b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589222.18/warc/2564552a145faceab7fe7d80601882c7508f3faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589222.18/warc/2f2692caa0d2a89d1f7de27ed9c181f314dcdb4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589222.18/warc/5b65827d4864e5c34985dfbf66c4e56e80ae2730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589222.18/warc/62edbee8e6daca1f36c3b7007b7485a2ce8d624e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589222.18/warc/69f4be65d0d9d706301c5cd789e80cc2962c4f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/2b63e84e95b24a64ddcf369ef488f30f642ac52f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/3a28fb6bb4380883040724c557f6faa8cd559829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/4daae7884696ccf0e51165f3577b4ae39c3ea2cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/528278f5ecb4a0b89fd507e130c4cfa860e74461.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/5aaf9ec43a5a31600d064d1766a6bd4e9476404e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/aac44c2a9050bcc565bd025d2bffc6ee243c8698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589251.7/warc/345f8aac6263baf28eb07569500ae2d21fa97bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589251.7/warc/7acffc803f9608718bd874e5086df99f3139f557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589251.7/warc/88c1e570b6fd37236863943835ce2c06047ad61d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589251.7/warc/a0a062b8a2e08b9d5cd25b618fb18335efa41681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589251.7/warc/b9cdb8b8d07e1811bc3a2e4b258cb8d03d74af7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/237200afce776f595c9bdb9c23e9405de0cde193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/32903e8552f8f82187685a5bb60926f8a985784f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/4e277087393df5a5345579bb4ab31fc0c57049f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/95965cc6a46f8551d0193fdc56c9df771cbd7bf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/f4b2c57f77c2e61d7c2834240f51a81c848e18ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/febe4fbc34f21033f82ed82fc3ecc8098a95f130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/0a7ed0a824e3bd14931efa3d08c617da785de9f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/6d90b5c52efa0890f8a80f7522a7f1dd2f12ef69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/8ef6c02a4605918a8f5b00f04291156d8f12a9bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/ad28cc2c1e3e4c83f44582c49b1ac16854c0b41f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/b4e5f2c70790ffd540d484bd9103ac7a3832711f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/f2e02f91816e76d6c5f991edcde96dcf8dcad412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589404.50/warc/050f1df2a4b9b6cb0262e6c029b3128ba6261c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589404.50/warc/0e3053d109994f030a17178c913f2f7c42156ddb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589404.50/warc/500f891b998584625136c163d20135ed32cba26a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589404.50/warc/9040a8f7f995dcbf2e4f04da2ab5f3e9bd89b45f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589404.50/warc/b19cff10748530faa9a53bbe5f7aebc3739cc56a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/16138c212cac82bc8bfbd39c23c98201d329e789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/415eb19c50e43f8434107179e5db7555693a0a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/63201d95edddde65625f3ba71f1dee73f2c7b291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/809718cf8b7fe72b1f571f48a9425a820c77c571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/9c01c82e9c51b51becb20bd8cfa8c8dbf75271a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/d78e16085bdab40b4ec03c8862d67046538db19e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589455.35/warc/1f45a555335c1cb5e3c6e39865ed797b8ed3c0f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589455.35/warc/219f39856c297a218447a07a1077db192692bdc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589455.35/warc/41bb6c33a2400bc5b69be8d2316c2c9a9723e349.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589455.35/warc/9f04138979df2f8d3454e9ca8e4072c083f639b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589455.35/warc/b4052c2009e65ddc9a6dc2024ab0f65a10a4a44d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/0eba424b9548f8dbdb57697d6720309eb499eab5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/1ac003648058ddd82f866354cc61aa8513ae8f31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/36843d0db016e49356d565f3d56e938c9ca9087c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/77a0821772b5d9b9da7e167ea9e36e85572d5914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/b224b705fd043424e2a00703d35d53c707eef2ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/e17eb1559e5e6cf3157e91e15903ee5f30e2eb9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/1742c240713d213fdbee9e043c58feffd976e5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/2a38d085f81d167146ea1c845a42f1aa147932cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/9980d328ce6c857245c405e2a85ebd9b05d65f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/d1a19f7f4d1442eb4ff38f753fdea54c5459506b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/db403e888c5942d2df38437cc1349a3b0cfcde36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/f2c9ff2884fe349bac50595c341368b1cf22bc22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589537.21/warc/032e6c3332666b53586f221c7c7cc159b5e09a8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589537.21/warc/761ca8996058abc8f631f671894cf54730da21f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589537.21/warc/aed36ee46d2ecf82b8785595a842919b28d73532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589537.21/warc/cfff23ebb7acbf2269953e8c37b58cfcca793be2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589537.21/warc/fa700d0f486e33d81b5eec04e9a8cba27db95e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/7728c087bb2ac5de63277af355a82a657a2e7c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/9901d3305e7f8b8cc3b7d398030fc02dae0c8549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/ae66a9d347b6f92cb9a572ef60134acc026af3d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/c712fc82c570cd737bdf8c012e12bf827fc5b961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/effe1e0fa7af95692713b6e5a4dc03e9aed4ce64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/f1a1e64e2d9a469ab7cc2b9ec34b9dc2aa4d8068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589573.27/warc/178865dc5daf318f11aa9fca8e4acdb2e066abbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589573.27/warc/4157afaa7a431169e130c4a1af0a05067695475c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589573.27/warc/4cc367622f7c0c13a2c091cb4e31814ac02642c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589573.27/warc/db9f9abd8fd3e1fa9fb1cddd28eb46cecf218ded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589573.27/warc/df919f5f8ffcda998d9f13892f24ef9d792bd1ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/469af8c208c3dc2bbc72787285957ac429433cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/55e99ac5ab92b33948b6bb8f5ee59321b8f21395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/67023e466d1033acdeea61e97dd805ad0beabc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/89a8700e04ade014ed662f4af5c80c8a5be5e890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/e0c51384acf32fd62ee93f59c70b0bc92550f8c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/ef8daa77577a45fe355662d4f92e719b6ffd01c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/001001c3f4a69dd17a8a514b3002abadb7036e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/46a424dbfb1534dd9a6bd7288ecfedc3b282dfb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/82a4b9e20bec12fb22077cb353ff3a14e168e64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/892a5df6835bf1f32fd4ad13d879fa4caf8a441f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/aa3401bc52f7cc5b6c3ba7bb415035fcdb727d93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/b642c80e85a432cf609c518af6b26b80be43f7f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589710.17/warc/340ae28648f4fcf27383cc800e812290e565843e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589710.17/warc/4414d93688f5f720872437401d3c5a7981961923.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589710.17/warc/8effcb91a5404b11d713fdeaf93a01723a1bacf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589710.17/warc/b6d028f8f82981b7883eb57d84b77f16fb9f420d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589710.17/warc/bfbf99f07db111cb6670d730ce17d9387bf8ddb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/0375ad910174c6c22f44b7d2ae1584120f30c9e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/6b3c1c1a3454140764009a4859f2050176e5cabf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/b467721cb8e751787b9a9b3e11b8836c2257e84a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/cce7587823eeec1631d5c7bfa2c3a82637f53801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/e9c1404d084e5361aa29457a4b8318265657908a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/ff50b834adbc2e5c9073277ec883333a8df036fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589752.56/warc/78cce0221b69592fa8307087ecad892c162f33cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589752.56/warc/7aee02c193d15e89c446cf93582329612c34331b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589752.56/warc/b8e032f05269d52f6462adccc2b49f510641e3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589752.56/warc/df5c53b160ddc36cbc66d69c962a77c485dce50a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589752.56/warc/fca6c886629aea70641230a556817614cb40b903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/03bfb36d72c65c10580ba0417135dee2f98265e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/12fda16862f9979f8a4bd7e09ceb7b11103080b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/2d1c078cbddc3ed1e3cb8ab03595283d16a3fb3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/52f9f28fedb246ea7015a3da7274218db2c117a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/dcaaf531950fdc0e0fe672a3907f4b2b542c6915.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/f1eb1696c589a395e13f86bc7ca5f11577eae60e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/0301f38fe39f04de1e708cc9f94a2f56d3a9b158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/41e64f4bcc489fe8dab67077edcc99e00ff6c366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/4b8c71596df1c4078f63e614311f706efd5bb7bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/8d0ed0559342023a13354ed7b1f5ebe801a7f0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/b48bd245cb13e181e3f68ed71aa98dfd025d4b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/ff06845b551e2205c5e728592c39ac9bbe425ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589902.8/warc/1cea8d5dc9b4fd91b26afe3e173f447b1fe3b13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589902.8/warc/3db45d28d40fbaf207edee58a09faa625430eb50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589902.8/warc/7d72743066e86ed718f7e776da874f07a01a67fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589902.8/warc/cb421a613c44be706a31c149e30056ee7aaaf65e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589902.8/warc/d901d947b6a19be2d15bd3117ef6f5a647901c8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/0d810e6daaaf6b2fc1ec0bb647b3111365d2e3aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/0f4109d6262e33f0b51e4758803f2f60f8af3afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/66e642cafbc3ee15426895067889dea769a4893f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/79ae8e0f126e410e67f0edabfa203d8e55ee9046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/7e2d7521c588f261802784412a5447fcb386ce67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/dd7a2dacabe721cce5c2fa11dbea02e522e8cdc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589980.6/warc/1e0bd4a4d4850062f925aa6c5335830bb38a75f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589980.6/warc/28dd7b37fd9e6a64a219831fde47b7de87fc5b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589980.6/warc/3d81ada008ffec1beaee123ebe1474bda09fe675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589980.6/warc/a436a418caba6df607bb11c35d61d1e46dc0f76a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589980.6/warc/c90332f71fb0dbf665f0e93524b9079ff5914a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/102a7ea62eb1e64aa5ededebd2239bbe193bb155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/6cbf281ecca2bfdea03613f95543b82c11376db2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/99e433e0c42e680bcdb78ac90e97ba67f81ac15c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/a5407683d83bcdc15430716d64f2d624cd0e0e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/e95ac173a7f55f8261bc80e926b0ec503545c87b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/f205aff7660391cbad17ddbf91728a8f4e4fe0c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/23ebdc7481c99e190b2997346f2826d7359c6797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/432cfc77f17bd83e45c997af2125b228f6488c22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/928290657b47df87182156e777ca10cd07b6cbce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/b7c741829833a794b614bc9b88363491c92e58da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/d7f33e1fa6a4d9842dbfe85d37b0ea4d08e2be00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/ee4bdaaa30db1d533c7c23fca7a0bad106373d96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590069.15/warc/594875929f87414403f5c10e17a781e6d701472d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590069.15/warc/8c6591a80afacd6c803e2f344894fc574d37ccec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590069.15/warc/a51d06ae67388f2ecc6d3512e3ae3ad63b3b930f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590069.15/warc/b4c95438160e5574eed26c1f01a252818942e2d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590069.15/warc/eccff9236234ae841818e7b6bf87fb62a776cdac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/26185a82fd4e8a5266c5aaf55f43799598066e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/9135d9a590b0a4a22bfddfdc507e200743631335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/a6d2f905fb48eaff64c7f493459eeae57479de2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/aec184756e8073c70da12c10a62505e2b238ffd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/d699127b397539a29a8cfe2782c82916fbf434ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/d713c49f1ab0dd7346073eddb0f95bf1ba9345ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590127.2/warc/6466d617d736ba7bfb669b51c95c1a3f64038714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590127.2/warc/6542874da4b896ab1ec5fa0a2dab447b98f29038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590127.2/warc/6cf614240b9a937363d02901595ad5cf91a00e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590127.2/warc/9dd5ee237103414efd9da8c6ad7b8a2b619da96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590127.2/warc/a634ad7fab5bd2097869f51c803bae4d1d713f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/3cc236bef56559d6170a0a84cd4c4e143dacc967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/6555930ec06d4c6fa11c2cddf35b279277bfdd84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/6974a01959528c7971d8a1a7362f0783d0c5a11e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/87b5e7e115cbf8ebb910fd9887274862e47e17e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/9ac66153e318ea7da7dd710043991654231d2b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/d520f2f683b719a84f98bc8bb23436969584b705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/03c398b44bc0d533fd89966949d1671bb76131ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/238e977ad5dd76e8f21913be2031c3e4075189bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/24c50f20a48179f28515767cfdf9852bf0aa3fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/6c5828d4936cd088a0cd5d747a6d1a8f45dcff51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/a84142628a8015ed18af74914c10226b10c59860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/fa1ab486067f90da20d409675d641dd3678dd767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590295.61/warc/1140babd16aa129e344ab4f3d4c8c981730b5576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590295.61/warc/17232d1538a1863e9919338a6051c252619dd804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590295.61/warc/3ad9f05673a33204ce20594421507134cff18959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590295.61/warc/a34b632d5ce52c37d38305b0468c9b5d116dc676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590295.61/warc/fde85ff18f02dd7f16442d2497af26b2919d636a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/1964cddc90c9a8e495ef7d94813983c2648cbfec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/38f2665cb96e6767adaa3e53bd51b6f87d012b84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/a6c33c3f25167e4123df569b4e78ab74a675c91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/afad1f52d6173680fc90798a1bf6d411967c558a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/b5654b22fd86ac2ac71b9a29db4cf4fa1ef6014b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/d1a9a0ef52c6f1eb6789e5054f15e729d7732c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.25/warc/28b79b84cc15ac22e25f33c990eb146ec72ddf07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.25/warc/53d003f05d8ac13a9cdf8ca7cc4361c4b9c2dda5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.25/warc/703c11cee4b26eec699ccd2b116cd0985e390fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.25/warc/89db85c9f3b63f10c51b7b8079694c91118ea36f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.25/warc/cfbaebc335447a7f8659768284b37cc8aaee68cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/07014490c0329c00230eed58ac77cda2ee737140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/274aa94028f58f2bf4537599831ba3a6429dbd06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/72e242f0ac3a6c614a95a26cc69bb30ff51788a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/9ad44651f2335f4dc043bfd7abbd41cd29ec3005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/9f34189354e5b11e368c182e5d0262de93d9c292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/d55bb61d6ec4fea8e062d6c5e557c7e04722e4f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/52f0a3b9f7853b13937eab3a29e6a4affe03ff6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/90e0e953373a64fc1a47edd4db82349fffad6068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/b2ee6d2ef7d827f71a6a45a6f887a191a67a3dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/c419ee6495b4b2a37349a820878084f55a249765.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/c74dc65b7dee98e819799b9c6e6cf2f10cb3f167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/fa7eebade50c2943cdd0e6bd6989b2702f9f2a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590443.0/warc/34b0ae5bed591819b2dbc3642b7f3f42ec03c247.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590443.0/warc/8a2b83e56d5d4f3524ca4e7d61bb5f4e1cedcac9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590443.0/warc/9965b83eeb5960adc3d76849ab804c51012e3d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590443.0/warc/a8c39cf6b859f58dc26f44d1c2b89b7f829a76b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590443.0/warc/a991e1b5ec836304999c5599b1420d4ff61b2277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/0086cb57b38c34b631aacf14cac0142ce7070afa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/8501a328a20d93148c1b0a5c27d377660db7fcef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/8daa5a2fc05b50386423e7730ef1d7d0e3a1ded7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/9944ffd633b27d91ea2ca1041c2e0ca0dc1e0581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/cf22ccdfaf680f1aab416428c7c9fbee8402e085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/d7f839a86ae68673fa26962cbb121d588bc7b84f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590559.95/warc/12e6a73fdccb9c11bb0e1e29b0aa00554b9dd5d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590559.95/warc/240a8a0b96f7a561dfbb1eaa01499a0a7241c2af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590559.95/warc/35dc143fb3c39995e25ddfb6763bf4adae7c4a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590559.95/warc/866d8d9a0407ed412efa66747b83ebd7de361b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590559.95/warc/be83734f403d32fac15144315d485147b22ae68e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/21896773d5b7179e08c2ec1eb09d5582468860e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/5b048baf5d21c1fe06ad023d62eb74b5042716d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/899572d949b530b11bd821098233254ab0a333ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/a0a603e63ce15c1e05ad41e23d434d208de7cba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/c136e291aab78d2cdaea1e2233eeef4532c5d087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/ca7be64018dbe3ff8105902144c093c60d04be9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/0da5d298d87cd1df20f04fa7119835c3ffc9ae07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/7d31a6076f49eb31c2ff3485b4eb8f56f660ccee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/88b0c5c8209bb34618e6b44d13eeb661d7643514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/965cf522088c786971334861dc511834dccf41ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/c4b538fcfe8ed71be2d950a9c1431520c9d36dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/f1885cd2938baa53745a454203ce0726d7f76257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590866.65/warc/6210513972d62d3a714f0ae94f599c610d20cced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590866.65/warc/7e208b7627c48772f11b36421131c72925959d95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590866.65/warc/8eba5b5ba132dde94649ca4090e3dbc90f15e917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590866.65/warc/a777d98e3c62412b082dd6a101b8c74fcd0e9c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590866.65/warc/c58b79007eab9fe8b7665e7b9a122cc8e2471db8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/09f0d8f4b03e0a8aa3ef0e38ecd4ff5874e94e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/215ce9a75d2aa23120fec0a0e11e4ec9acf5794d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/27e1f4dfccde47cfa09f89de501fc5378554886b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/7be8e43b87665088ab0b12d26f24bc3ba3d833b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/af424da316f58c58ff36c49805c56837542bdb2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/f78865ae8e4ab57bd0cb3c51dcee53aedfe7da60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591140.45/warc/17eac5278b319a41970edd2af984ae6cbeefd13a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591140.45/warc/52edaf2a2220127ee59348931d57119a4699fec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591140.45/warc/5777215a242a46e83e4dfc80cabf52d4f82098c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591140.45/warc/8b69b6155aa14a439f3e2d33c5017c67bc5ba72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591140.45/warc/d93a4ec063bba95e7cc83e8ecc1536c87f3b7777.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/2acfe486afd49d1d628523f73183ef6d295c91c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/4252f677f68de7ac3df706b41c0e59ab334b2a16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/5be3a145083a1bbcce14f9b38efc3a0a5cd05856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/82b9ef61201ba08d878a3bf8222deccb3eeacd01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/942ce08f4370dbfeecb12cb8c9dd0833e6662ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/9b58dd0f595c4977b0532e5cdbc4251a04a4c985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/09cd55e7ce63370178df90085ef8518479df58b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/674b333268bbd9489dc36d02677e34b3627796b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/aa3027da0859e200c424342c5323ca629d1aef9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/d18956d49df7c089602bbda0b417af17082f61e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/dcba6c3e6c2847736972b92a1920fc8db0d60a4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/e1f52395fd1b7a7e528fdf03d5bf37e638f98dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591296.46/warc/16b02844ac4ef1b70e8f4fa18ddf9f5e3f1900ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591296.46/warc/35534b7738b37b7fa32be6d519bf06a42e3a48e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591296.46/warc/5dc31146cb57a589e7e4e22922791f483d08cf9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591296.46/warc/865a5d5bb09403edbb67c14c9c4f759b2ea76554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591296.46/warc/952c678a1f32aed9d54f523f25351871f81ea262.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/0986edbecf3ada9329cd50d1fef02f23234ca84f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/2a3527cabbba0bf4f41dd8305383e737fbf41a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/60bf50e134c9fcf24f5a03eed398e7718e156752.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/797de6fa3824183061539c83db33799ab0a85436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/bc55320bce04d68d202a945e0766e365b90cd974.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/d3ff21b7c787510969444a06e74e9e8e43d20ada.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591455.76/warc/0dc248f9ae9d071e1c818f3fda486cb1de0a1aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591455.76/warc/368581b4249bcd78cf2eef6d88587d2373019a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591455.76/warc/3d054fde5fbe9ab26a378a998a6d724eca4afe34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591455.76/warc/be05273196bf605438a8a3a1668a711f04a8513f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591455.76/warc/cf69479a9eb545c2aa29b23cfbfff4f6c9f36772.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/039b86344bb41a3fb6cdca1f2041c29b473fea5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/493720df0656880c10c9ca369862eb8f71f3b0e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/b63bd7723ac4e6a7b94c8a4161589bbf1637c348.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/c7c9620640005dbbf44ba4875370e80291359bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/d22f53c55931548f663ed1fd99264aefd7633aff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/e103f1e6131a5a3bdd4f179872eb8269098efd67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/046607b0c420d8bfeca2de5024a7a87404fb44c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/0fca09c6f8fb750210ad245aef3096253047f7ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/2036cca11332ad8e04bb00ca53373d79abe0ea97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/3b7678df6679d3603273448479e480a9882bebe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/60e5e5768e25efe13e848c23a22866726f465462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/deceb04882bda6d95b378a225e87abacc67c7880.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591543.63/warc/1a93873927702b4e4d11137017f3003a286559e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591543.63/warc/2aca71acfbdd3b6b22caa4c4c62fc71d355a6297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591543.63/warc/49f5e7a5a39b0ba01c750d514b1f741995522e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591543.63/warc/b1fc32f47ed36d68209a464c18c4a36841cc2856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591543.63/warc/c7ae4914db1a1c115498b09c5dd5487d36d76926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/431a73bf4d6fc578d015e0bbaea9b6623e536661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/468641ecb5dd87d825c0645c4cb33515b4a2d66c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/9f73f4e96bd6ae0fedee3db7dab4be3c8635f995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/c1e8febb2f9d3da1d5276446d5982d8956c08ed2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/de02f69166e92ae20f1f84a2d8ecbe3bd7df94c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/eb19a4853e5b396d4cd7bfa6c40aa1122a095b04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591578.1/warc/09ada2ad43ecce218ca276ceca083bb305b0b8ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591578.1/warc/40e33288be9a02d5522b0bacfddfd96b741b173e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591578.1/warc/aebcfd0b02973c106f47ad58f3cecce51880527c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591578.1/warc/ca8fecb424d6f5f1474323fa640c533a6b3008c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591578.1/warc/fcdac1da31fa41b2456cb66307c4e0f66a1061e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/71456f589686a65e5557336f453441612a8ccbff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/84915cba93d862013b8ef5892aede1152087dacf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/929fb372b3670a7276923d14cba8974e4eb76f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/d1a0c7112409a75fcda2d3fed2560cc0015b1548.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/d1c5afba1a5d4f1baece8db865bbfac0526e0aac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/dbbd3a1edebb61e0c3e24ac8bc8c329f46bf3bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/02450d6388562ba17544300967a33d3850cadcdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/233f52e3da446869373dde1989e9aa7837722835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/2f9b4259a2e845134c10f7fa9661360bb9ee29ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/43a6dc59a92871aac4a93081580876bfa3fce74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/be3dc7ff2f1f5925078dfd4e47e72d6c463d05e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/f23250ad3fd4bf64721c279f4b18e63a10f5d77c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591718.31/warc/2999dbed708e74f9abbd3d538acb94a7da2de295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591718.31/warc/472dbb074aa8dbeaebf6940d393d75fbb9eb4bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591718.31/warc/52044f65b823fdfc4067993f2378eca291a9d58b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591718.31/warc/8502a9cec1555bb403390da14935ef5922e2ea9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591718.31/warc/f3099193b20a0faaf189264f32cf80e06ede9cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/3fc538c859477fa74e20afbc31af4c04f6cdc116.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/4b8476310ccd62bd121b1c5c192824feffd685d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/57b77c0bb82adf1d1b3b26c39c01b91028473671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/6af6c60b87b54fe36faf471d7a79d0bd0d6c1054.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/72ce09d4c50b1671cfa7fc2a3ec92fcd1eda80d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/cd60d10dea33f1bd19eaf68418366e882c2754d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591831.57/warc/7de38280c426323cdd0cc636316c21967c21c9b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591831.57/warc/ad0023bf37f79db8a3efa79d3ec1b3b673a0768d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591831.57/warc/af4f2148e4446a3d1316db65564327555fa18319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591831.57/warc/b317d648fe004b0da946e3acbf2c72d6b077a3d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591831.57/warc/c76e266950853d87da731f2b1ab24081fdef5fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/321235bb43bf18e377753841e2d97283a3875df8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/b9ae718d55ff1c25ba0f8421f676e6ba5e972ebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/bffb495349a09fe6b37b05c2c749a31bad710743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/cf251d4129dd7c0974255b1ff8037377dc4fb757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/d1175a10f7f08a779c48b5bdb3476feefaf5a913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/dc48d46b207970c2ad13d35202f38ee9c571a624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/0a36c35ff926dae93cc4fb66fa90c24141142700.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/4c940803a3a0a0424170933543b1b41d74fc1fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/637aaca2f148d5d9c351d13ffd7d654799f06304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/641eda63c6823592504d4dc3d1e546fd3610cc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/68c8c1923087e88e70edafdb055b5054909c930f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/a067acad316805a5050149aecd3469442fa61669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592150.47/warc/4cb29e615c7330f4dd7b39c86ab5597fa9e532b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592150.47/warc/64ffc4c90db234bbcbb19ff051d56365b85b5061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592150.47/warc/98a386b3b9d2c8cf84a97fff5c05867f0f215840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592150.47/warc/a366bb64ea922649feeb69d85156714c922fc46c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592150.47/warc/de3647561c13eee848b7ab2635e1735216000bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/1b65df9827fabe150193ecce4b385a58cd9a1c21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/2309605cc78dcc919629c153ffebab669b978fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/32c4f97d5787e1bb7073c68fb82af0ee9e11e2d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/34605215a7d7708046232a9503384918f608a357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/5e958371c447e7516b1cb078f900dca32d25eab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/f44b115a7be111921e82c4de7034882fed7ac2c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592387.80/warc/1e0406b2b571acafc062876dab22dc479ec1cbf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592387.80/warc/87a64e313943149c783649172715de27f6be84ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592387.80/warc/a5d861afb93f51211ca122087797a9e5d9daff9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592387.80/warc/cb78d3165641d687259dd286bef0734517a6f934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592387.80/warc/e5fa990dbc240365e6f6437526369d9932d8343e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/56525b726960fa85afc462f83dec002538cd7833.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/88902591b882117f4cb272b49a14ce14f64cb5e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/8f78f39cd4c92457437cb5febfe8b27911469bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/ca6c88a7ed439a6c9605312a2a73e683ca2eb2c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/cbbe8ecc87aea23faf7034576fad27a9a36cfeb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/da1b97f18811c880461a9c08a1cbbcd634b7b9c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/0cada9072ff8934495231d1961985c2a537f2917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/4353969934511ad344882cbf2767f3236ec15276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/63cf0ca523d8839dd91602634ebf52a22939a7db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/749bffec006952550e9d18789e62cfeda85355cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/ab159bf6dc90a7a940d97efa9edd896295d818f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/ed78c9d54e9415a4f04347008271558124ba05c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592523.89/warc/33d833abdda8ba5efa3ac9a08a7794ceb2d03ccd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592523.89/warc/6c68e70fefa092b44161d8398b1a0b0197eb44a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592523.89/warc/81fdb28dd3bbd2fd71a1b0a888b7b353314148ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592523.89/warc/a74be61b9bbd5558fafe76294bb7a3671961f3de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592523.89/warc/d3084ef73cf1eb310a55b07853f53335b0025f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/208edd4a571ccbafaade43b7f26df923dc249047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/31cb4aad727c3ea348fa06d14e2610d4aa925590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/41220e57d476d7ff4038694970690fd0701b7e0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/7c9a86a067c0bd6490c0d49bb761f3c0c6fe02e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/804a189f79baf40ad0d62e115596118f8d7d613d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/86a3bf3ff58a2013cb64e37f50942c4736404001.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592636.68/warc/3f4b25b4fdd3e50c59cdb1dcc0b9cb5db3d8d65e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592636.68/warc/4cba89c258b0d442f993c08e894ebd73fb3700b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592636.68/warc/6de9e9c9e8c88bbd5e390479c12e323749c9d584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592636.68/warc/ac3c44dfcdc9825d34e4b3bddf43500f12085771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592636.68/warc/e898ac23cf43cfba86bcbed44e84969c7d328640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/33fdabf76966a6686d28554fa764e381861a890b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/57aeecd32549ff6395cd779b2b51e401ce91a9c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/7b89c462b5e2e478f3fa528963b77229e92a6390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/7cbea2f8264984575b0db3567bd5bb67ea27595b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/a9973deb7a60c7a7bb6bf62036baa347de47e4f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/c59daa6766b8691c70cdc99dfe1c815af20064d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/1720bd3481c19435d0492a37c3edae8119ce7ffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/46e8d417f45d4bcaca2c74439f41a764ece560e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/a614d5421a0cbbeb7032fbaab1ac9136f392fd45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/aa003b76876898cc5c1c96791855995ca769ec27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/c433f4f2170afe7109e57d632e81d8604573da9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/f78187f6a73ed4c8984cdc2a46da478b02ec311a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592778.82/warc/109a11d7e971365df7edd4e8d216d5049fd816e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592778.82/warc/485c5a221a700ca38cceb8826bf71b094ca5d1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592778.82/warc/54357ae426b0cabbb603846960dbb66380798f49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592778.82/warc/c2c4fdeb92f153cf03c439ed129bde5216227e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592778.82/warc/ddd5b045a138caa76ac6f52926dbc8917f97002e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/0863852c0639fd88e947c2e632849cccf3ed54c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/12f7905de138d4b2a7c4e7392b153d2ab0f8125c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/4f1b6cb611c602b1115447acba4cf4d340bb95ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/577fee7f3f89616e17ffd6811467bffe42779c62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/72927ffbcdf746b56bb89a0b3bdd35eba2d8da6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/f250fc7ff30bcb402fdaf4dbf1fe55c04b645498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592875.98/warc/0103caca1a9542984044c79d551d36945b6ce2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592875.98/warc/5d789182fc8baa5438346a5e545f40b4e6070255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592875.98/warc/62d460c9105e319da647eaf3198b02170243c743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592875.98/warc/9dcb4b9a790b5393ef8c91d592a12d71fd4c34f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592875.98/warc/e5a8f821780e1b17a5b35bb7e512c2ee0e11ff08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/109b6a6314de05db6267925781295a50665ef456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/5137202b4d6a662da142433347b4b1cb45201819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/72eb739f16d52e73ef6a456f6d5c03df105d9962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/a2e44d7f5124ff41fea140c1c926cd580c4c9afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/b0bb04099f24e963d9418402578ea788d45ed4e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/f094df6a9c5806b64f12c12fe7dbf6ce598b5552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/0578bf7f6adce499609df2d65697ddad2e79b351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/2b44609feb52588c6f2f104fe9f41779b24245e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/33fadacd50079a64dbf626e3594957513aabbecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/5cdc7e3a50581b0ec50213b10d294eff400d72e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/c20df0c559b68437f6b6c5c9fb8687461c5d8717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/d6e3caeae36ae0ecf67a4e8421e023b1e318e41a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593051.79/warc/31a457f9744d63cf996911453ba8e334864f5067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593051.79/warc/4f56ed1c66c2f814571c9ec469f9932d8759201f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593051.79/warc/623b4c35157bbea59e9dbae9bb6ee780adef0eb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593051.79/warc/793a1d6cd094ea34cf9312386632b3d03885ac64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593051.79/warc/affc44b25611961f991ad52362e870544a070ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/055ed2cbee36be49e56ba1b5501eef7335ebb563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/33b428fb6c08e80014cdf9343a65e3fd92f5ad3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/68618e4e3db7734c0e6b1e4b91feca7e1c33b7e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/ad35002184b731483db8c83e60735eedc2817cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/e18b7f7646a8d481a250b5834774856cc3339972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/e6ec61e7daa66be03cf29c059a34b231e1a83699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593208.44/warc/048b0745a8e90af110c4748751bee82067530614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593208.44/warc/21b88d1a648bd70d4f4750b9ccf079c2eb2d9cd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593208.44/warc/39210569cc6d80e29d0832588e272a400566ef27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593208.44/warc/6f20eef0ea057fc10a52adb08c8bc5289f33d1ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593208.44/warc/eb3aa36b77d5ab32960932942bc599b620284108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/2c9328b8367a371535fe618a08e0b483207a5609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/406b95a150ed19f59e1de844758c6789b58c87c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/98eae94040fdd631dd73152b67ccfb0b9168538d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/ba37e7c863c7cc36e58404f116060a76de8beeb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/bd57aca26e3c1790e56644fc7beafa4f10604fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/eabc95bfbe7e6d2dee8486498a60665c65e0f192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/3e39f03d7626463f20404a6e81e2cfb6cef869c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/73bc1e32282f7ee2d31f2f8c37aa4a1882d2d163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/9e35ee71004d83cf4dffa0f955cd40c4a041e18c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/b767f4e2413670e18644c6fbae6fc58326aa4a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/bfdd97be56593a7fd59331d4db62956dd91f6525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/db069fc3075862afebcc4a6f09aa802e6effdf05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593378.85/warc/17753f6a45c25c21bbdce7e0e9d381689da52f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593378.85/warc/7d6ca4da18dc06df26d71538845acfd644b534bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593378.85/warc/87881dca97b15c7958e17aca8ac199902394aaf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593378.85/warc/ad692b0950648cab67cc521e253855a3e21fe688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593378.85/warc/d8b65b6a5100624b2ef9c4aa965a0ff108af5d32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/23f97d624cfd3c1d14563337f92e28fcf414fffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/50c9fee03f91e1cb013806c7f5f6487383cdd2b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/82347fec7c175d0c8199af89758c0ddb039e06ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/b6862177c10972364a558e6046a35c9fea2077eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/bc719752834586e59b88a08698962acc8cd091f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/c3e12751a91b62c3bffb3a73560c41d1ee0d7335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593586.54/warc/21f78cc5a6603c74cb7bf59efbe462d68425c9ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593586.54/warc/3abac21edaec8c3bd5facf5c3b15e60a6479a98b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593586.54/warc/3dd4a15ea170b00567455f056c2243ccc21e5ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593586.54/warc/4c523cd9add40d40d7f6a3172121854e10c9b586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593586.54/warc/8176cd840a2b014c23be443afaf1e59479f724e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/58fc2eae38391221e95061f4b0ac34ed83355932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/59b26ee69bb4b1cfd511030104c126b2cb12313b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/a0f7a0a272f24d07cb6cef9a9969458fd56c145b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/a7c82a5df3d177880d7beac14a554826266e3ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/aad4ce3127b857aa581c803df6650506151da681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/ac95688bc0f40186ccd1dac20e93762eaac95f70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/1c98a83bbf8058f0afd0fb5ae3db9b9263864327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/52ae1d767115fca19c05e19b3336e61a4d6f5b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/6a961a8199ea9bda29fa1ee7fd0d24abbe9420ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/937870d2e8802b15db8e5b681faa0af2d994be28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/974be90c2483dbf23ad67553d55ef87481f22be7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/ef4d7a7bb5e1f2c50c68a8d91d16126ccee01b59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594790.48/warc/0cefe124c68540c8aa971484a1f45e2d5be2a4b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594790.48/warc/422e9147a7c23826811fb5538793e2f765371b4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594790.48/warc/542aec46c9b7900f50792946347b37320aa6625e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594790.48/warc/9638103ca9505aa568cf8ee23024c767f66d8a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594790.48/warc/d1257d29fc7e33a06d0862d7264b74dd85b9c3af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/1fcb066881d08062cb59d79c6e344d6d66cacfb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/852bdf66eec4a40a584239e674a95b4082841430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/ce039a063e788d371e7c94b91849141836245d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/de3d3d4dcb1bcfa1e080fdd148fe207cb3bc7ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/eb106f60dada954773d62c000bb59db94cd0a6bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/fa067813c6fd142bee1f69247426e02dbdba6b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594954.59/warc/0a99da1d14c7c78b2083fe7b769488dbdea460d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594954.59/warc/0d19a2f9115711c00a212b501ad8ddf2b32aede0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594954.59/warc/10abd4dea9712fa41bda0b6193bcc58610113f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594954.59/warc/7c078e52ab60a299bfce6a4cfd59dcf2e4bbd9f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594954.59/warc/9ca2e7288e33bacba8cbb905409027483d76af00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/36e4e920b8c6069473c29e55cde9d38470b97239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/3844b69cf9074322c2412cbaada5e31386322226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/45032324afe12184d38fffca537d5e9c12286e98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/884b2543dc7c238d1ecb5c795aa64e6ed820b453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/8b275a2f0cb83db4e3ad4186c315a41fd5f810ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/8d22fa3baee667ff19ab491f68d4b018f8af8353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/1aee6ed3bfebd25b1900d7c0a9008dc14e9610de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/35062030394041cd863e39067fbae8856f5b883d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/819260a19c1ba8e08fbdb511a0d65c0019e21307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/aa415c1c669c88176d550bb1cc04d9d590105bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/b56894917d9f77c59338d429d1d2122c03e4a45c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/fb01036115128073a2b9dceda4a07f768c9e3874.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596336.96/warc/142149990d97647ad3b4fedc106dce624e15a1f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596336.96/warc/3a8caec71e7db800acce07d346056597b8e22e21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596336.96/warc/579e49e2a91d163094da357c109e20be88e4a634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596336.96/warc/703afaf7891552ac541101a0412a77e65d231858.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596336.96/warc/dc281a56c5e0851c24a4aee56028356e2c87ff93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/577e7fba098ef0fb4f1da8ee9c7fe8dd65f3b26b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/76ac5b83969c366b5c05a67de5732fe739cc9a22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/b370145958f5cd6ef2d6eef66700c75574ca164e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/cf524b9220ff1ac482b8f01e41d1200f6c163a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/dd7fecde0a3796ca1e7fbb2f33aa520c02c2b73c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/e5b7e735fbd48d9d9d4fe02dda8a0f0f46dc8b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596542.97/warc/1871629219d5a49a708a67cbe7c7749768499c31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596542.97/warc/461358f6ffc0846c4892affff517b5afdf2681a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596542.97/warc/5b72816a409fc2980757ca0a4f40b90630b104a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596542.97/warc/67c80a70fe618fb42a6a619a7c49d3a5cf978814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596542.97/warc/a5e1b0d72004387582248b6e392f8107bc23c20c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/1ffc93d2e4a9ace7584e526ad8c1c8a98ffd23fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/56dd90e18d716e42facc463bcca25627fc98b3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/a55e12c53afdc0b064b2b514a27fa7f6b349358b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/c5903c07278ef33a7334bea2f3d625227d88c536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/cbe24ed00c7950b146c6b1e74f010c3e4ba9746d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/e1893ea23b1ed57f63a0719b08ea72b97bed6f6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/19cb14d61b5acf4cf334690353e80bb72b32e5d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1d050159a9f4c879c124961d7d55cdd0c9edf515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1deda4bbf2655eff8ebaa0a87f9456b565baaa3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/254ebed008e4d7c80c5932b1233dec7424d2a835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/268d904f9f5fb060deb8194b4fb3d50128b8a1c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/2c61588dfaf8fd0493436633709044e7c85ac91c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/2f1d9cc8a92f12fb4e1220cbcfdbb752add6accc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/2ff9d3b7019147fbb3b6a7df3f70206474816340.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/308ee4b9c39c9ad11e81de3fd4c407c82642f619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/30fd3007381643959124f078092feb386189b38c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/314d31251627a488520e19b345a9fe1ed1f36916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/3275629ad2551700147a64906fca34b4d64abba3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/38252e2f97610fae4c450b39dfc80a73ff02c050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/39a9de1bb05b3f604f411f1c8b6145e21cd1a574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/3d903b5dfa3e057b59e80f2b9cc29330cef575f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/3f2424a84d6676593fcab718ed530a7cf9e0b799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/43884a0f45b23aecae8f8df4d3dfeba943ebb329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/44246741c5cef6013e1e893236020a7e2f921514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/48d79266d1115505d032544834f7a337caa937c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/4a069f6380bc4100842e0f6a15d1c1ffd84495e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/4fddfc3b27d9a0a8a12efabd7b0cb93c03921834.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/521b05e692db4384e847d1b574ee84b52604832f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/523a39a7742d68aa44bd647ea36e3c986e249c11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/57e4cc170f2fec0913bb3d6fc7bbee61897ec0c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/59536ba150930604e26fc3f9b91f5743fe527dab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/62f79ca3cf687efa7668df0ecb2a2c1624f45291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/66ee9248c9e74358ce88ec82cabd2f67db84d61f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/69e77226b10d22f75531d2961635764c76861c71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/7ae3f43f78186f82424e1edd76fd315048d5f148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/7d1f96e6d26b9e8915362c285a075c29cfcd2397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/82ae31e5e1e6ad8d47ca95bfdd8624f99badf4c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/83095fb87d11c1f5b60cff175c3d6c1eb1f7f00f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/84dee9e5e0d6fea5a141ae02304430f506582555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/8548e14fbce1390ced4816b5dbd83612121a2b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/881f4618435e672b01dbb107404fa97a86855fa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/8906e3fb0d41f4ec26f32b5fa4bb0ceb86acf169.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/9110df6ac8b0c4ee16889efebfe674d0025ad978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/92073b7bc881ce14709f794e4d1f5a0c7e74e81b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/935172dd15aac4dfa8e996311db798d8affda9fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/9384e18f502ceb8964ed795693fd81b304c83a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/94652aba6bde1beb394e2bf75e3154a3dcbb99a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/98b4edbe8e63d5f7ad926f1f86564e3dba060ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/9efbbcc8f62f0924b8694481cec668f178f503a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/a0aed8ae9f57c09e6c23fb29ba74a15337e8406d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/a16361d9ba637155256a9450cd570095628e3451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/a88c3928e637d59e6917e072f4361759855fd355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/a928a7152cdef90cc62fd230c119f173ff1cb3fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/adea7c66dea4c1c1b79b45d69a9970abbac02b41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/af0314ce1ba85df0af1e8070e69da005824ff0f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/b151d1f4179abfc7de1875dce6ed3d1bbde21c4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/b386e70a1d29f9414a375532c29c3a79f6feda04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/b52a09fdb8b624b638a6a8d7ae54caf674770ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/b95b04366bc7c33705bbc9df6bf5bd783360e8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/bb748edea176784501fc76dd05b5e32cda147b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/c0242f7c779ce55c09c3dbacfc37d074d1f78ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/c5f46f913dc82c0f8252d08bc88ab6e4fbac11cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/c6fb4f15156860af22ea8b9c245bdabc599f12e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/c931ca54a6a369c0723e93fb8267e9fae9302ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/cd35a15d8a8779e13741fa6b90b341b7624332ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/cda392efeda4feaa6929b0c0300ae4244c5fb66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/ce51fd8bd6bbd2c870b2c09f8b9c429d888d7944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/cf5a9aaa473a46364c667fef4ace514e63765052.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/d05f0b5a435b856140b78a45dcf4ae45965c46d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/d2e92bf4fa732721b94699ddb945360f6ee2900f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/d8faa1c00778061f0b6a8b5420b46321c577f8ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/d95c47cfee49074e1d147c8de1a72dea13e0e4da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/df586fb5cc25f4115b7731ad4425c82c1d3d3639.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/e08d6da169af8b488359a32c2606fe6e94136c68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/e503f4a54f9d9dc8b1acccd5308c62168bcb7fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/e5cc949bf0d860d671d20866db8792d749dee80c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/f13442f8e396df7018aa2e032fdad8d6500c28d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/f80da5b9a5fcf4f72491a7a87c7e478556588a25.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2018-30/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2018-30/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2018-30/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2018-34.yaml b/configs/cccc/dedupe-url/CC-MAIN-2018-34.yaml new file mode 100644 index 00000000..7289b556 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2018-34.yaml @@ -0,0 +1,737 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/014603c33d36868b99622c81c2f1c8d1c37df884.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/0af6972528fd846ce0286c40dccc22feb0a5ece6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/0d2889ecaf2a182a8efe6aed36f1e04767f588fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/0e5739d4526f9fa8ca0cd0687af731fa2489ef18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/0f73981191c29a4ceaf1faf4d42fff23f4f3ff96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/13285b3678f982f66edc6e0282f4a0d47608d0c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208676.20/warc/0bfd4f8b296ab815185730c0740b389045243c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208676.20/warc/5a22d91460f95f6aa69812a3ca6b3d71c369c4a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208676.20/warc/615418dbec465140339124423c7454f6a842cc5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208676.20/warc/7da5c304e2ce17ebe40836a10272927ac3f204ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208676.20/warc/d1f6d64248a2ef744920413cf80641a4a0fc1ee6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208750.9/warc/0e39c90a4f7e33b4596fe5f8285c50b52b3a6ee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208750.9/warc/4fc1bec5dab088236d43678822a54a57a13c282a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208750.9/warc/885e043c1d1483bb156af95ffe1585282928c7e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208750.9/warc/ba4e7cd545aafff0a60db43c43363b3bd22185ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208750.9/warc/ee62addabe6e51d71778ebb3e84d66d602af25a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209021.21/warc/294c74c90eaef71cb6b862fce4e611b33135891f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209021.21/warc/499c0199340a085162c68491f958194cd22e0778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209021.21/warc/ba3df43f631d89b6f8dc9eed44f4c0537576f184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209021.21/warc/d8e68e2fb17ecf147d12477e8e3d5580c3f84745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/0800ef4e5995a582b717715d1c6f5e5907bd90d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/40c20a44e8bdac371ccc1996bd8196f72574f4ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/8485faffbd70d1dd1f6eb207d5c79296c572b4ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/93bfcc86993e2764527ffb3235d46fdfa65a404e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/eea5c95a97d9df16496d25ee51ea6952e9a9822f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/fa6b087d08691db0586015a00d097a4682425208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/fb8b0ba85be75ee3290b9a147b2bcec73dadd702.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/14f8e8dc88cb43cf11dc3687502327719dae36e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/421f1ae19a57942e7b57cbc3a213714618861bcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/5a29797ea8a328141b7f916fbb378ce8dfab7270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/6732778ae551c92ac1b281bda9aaa893c417756b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/7e474f0128390b5d9a64ea00fea32f20601e20c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/aacebefb508b31901b721a9adaaa654ffcf27c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/e8d05898c6828c8cdcf7aaa9ebfd6cae7b0285d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/15b2fc037a3a7cc207c8f4dd5b23dbf4fee83b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/58ef5658fc2ea5b4951972cc4e56f396225f790e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/73582732b56ccef775181ae6401e0a67f2d24e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/8e1dbce81c6e0fad03b45cd7e1e9f29108cc0ab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/fc208c6b49ed3286fcb1522e843106f3b68a7e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/ff138e6bf7edce3b020b79324585c3065d3da9f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/5ca5a0b40bdd9083e009a2c1e09094aa2436a703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/64996c0a9320649684ff9f72ce424d24091e6473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/70720b64f097b0c7be86b48b3c7f2690ae97ea3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/794608eca3ebc58b6a1e24fc6fe2ddaf63a8d2b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/9c09eec19992ce1d7c979af61fb4c0478c6e7f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/f1bb2516e1ab4365e6ffacbfd2d2a5ca0f533ef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/197efe8b47ee2f369e40653b36c34b50bf505c60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/2238919622f20d741b422528d1739c6bba1a7004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/3ed9cf34612731b5c0a493d1922362dd00b53b98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/422681deb0eb0cd0293e86e8c36e8677b167420c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/7c666f9b770e15c19f3c48bc7034aaf047567c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/8d0be53aabcea6895c9972ab84752fe8737c8704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/3dbb8d65885872d5645917ac7cffad63fd01958d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/3e0964a490c5d6298c4c9afc798479f73506bdd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/890ac95585474ba2ad79b3ebe9e7536f12c75019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/a63c9d176cfedf2b950e5a92f8332993e259dbef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/aa66a4733a491d83d485d2c5d1f4c175ff2857f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/d3edde9b715ebbdcb01e2d6235bdfbba3e7a5cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/d68afc27acd6f9cd278e0b5462ce415a41d6d635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/330c3bfee36949d595876f2ccc73a3e2dd6e35d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/7c5d288cecfc8019e65b33554ef635fc2072b722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/810e46eb1f0c296b5f3a45490c8f0c10dfa05a89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/81466973ee1d7ef8ec4d3e05b85374e1dbbc764c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/b1bc1a730bf84057fb177d79faa290ef2ab4a9f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/bb68a66347ebccc444cafd8fdc6af33394ecba9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/d62149f43664b5641038f14cfa1305814529e344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/39466844513e1e379954648623763ac799a17188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/51f7f530b10373dee64916df09495c9a4129e298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/5aabaddab528b5ae085e15a214ef13b666af9245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/91e850bbaef153b16c24a1e491f37bb75c03de2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/a3e75950a56da8ecabdb0ddef6d0451f349eb348.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/a89aa833724e596ca9be8a4c07a0d9b2906c41cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/1407183c4e46da1333237a45792f22a111963f29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/1cab2c18b3099e744ccaff20b4679173f983c56f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/2a1d1f4e5c0af92008e262bb7a18b074d467a39a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/994c2e2d234994f362229bbf23c1d78f451a0d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/b0cfb48e74d6ae2c17015859d0221505a250758a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/fe43d766f55932b9c3eae30ed9d4c7b3ab22dd9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/2d41c7e36126877dedda6a184d82281d3bf7c40c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/5511c5b78cba0680b1543f2b1375f439dbffcc8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/6cd9b564357c6bfa47da5e97d625078fa409ec8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/b111c7c5dc2e1f872f1306c242e475e1cd75e30a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/d5ce460161dd9ceab315cabd244cf1aaa8f25e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/faa3992272771b6994132e668594164985c6943a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/264fcde4e6a1806b2c61729a9b7570fca3fba959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/3587841353897250a97e4f595892faa40b3d2fe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/574aa69b1f31740da4af4fe7b20b7d4890805c4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/90899faad0e7a424558e5734177ee8357f06c9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/b1ff9916d73b63d85aef1e5adc9c53dbfc1955f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/bfb8ea5326de72896871b23fd1aedde1257d1ea1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/f1a9e63768d9aabb2b9645636e768d1ae32c56bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/100a85f67853d243243317026d28087e8bdd552d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/1761ab44ec5ce3bcbf13a0fdda28f3f011e93c62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/528863784058dd2b36a8da20de86da84971d66ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/7402111154b2fb966ae8966de18564b1e63b55c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/92a79861e4b1f6351496329e03df79838655e261.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/a0bb2674a41cbdd08fcabeac5666c3e0d5a7836f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/c7d4afe7c1a56d0dac83b97c4883fb33f2c9ea14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/777ea6276e0415d9aefa09cd04ae13fd79c229e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/acf5ceba419799e3d72487fd908a053a24ac7ea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/b7bfd49422f5161192230904761e9d8877d2a554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/c8d22b656f4b7941233d1fd89c141385e9751219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/d139ea64ca8cd0488ba519282be4f4d51edee181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/e8c36a336f249af51ad0b6428d499c18e66a45a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/0c37607ced959344743c178a931c52e04692c5e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/19cdd052a75a9e18cae9f021091f51f66a8d1e96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/2f9b113606772e1ade61925e811427e175846af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/736ae3a50a454d24f223dacdfc129665875f03b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/beb133f4d1a44657db3996ca26026e9f0ca2f7c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/eb005872278e50f4075d778ba3f5a2c7a3e4d8ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/16e527342f08352caea815713d3b6fcf40053c24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/4350671d9bfd58bf4b068de37eaaa852ef320bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/7c3f9cbccf371fb81b2611046065ee0085b0c14d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/ca0db36f610026d650b400b182430133d8286caa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/ea8dec53fc6d0408676c1b634493ba60532ec613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/fd4430c592f8aadbb0accda15cc3a105391882c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/218c5b20b6de8e759502979734d4fbb72e1b3db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/31ca793e1261a12571715f2d483583827634f763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/59b201adf2bc7eb3e7a3c448412d3e17a9113685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/d52008e6b17f92f2b4fc3a93b53bc3a3e618ad2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/dda9c5d424fd341bc599b5e7c6f27b0f3b06a888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/ecd67c93260f2d3c6ee38f4eaef3736e8294ceb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/ffc5e20b1aaf45af9e0784bf20d7423b765c69f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/27ca1788f50e1d370da31e4cb83baba2a82f0cbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/2ce2596d93469e3199a33d1bdca6ab687a71ace0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/4a344cef5dd74efad086a530f3d76977786caf37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/6c4ef12a8b820451dbb371c81dda121a35faa1ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/8ee96874febc3a579890f19a32b390d9b50a9d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/91c4c7c0d05cdbbd9bc88451406994c00ea5d7e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/c516a57fb06b698ace42cbddc905470844497f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/03b7391a04a2b60dc6ffd97a7d74ab2b8a47129b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/078e9e76f5609e0462a44e100bcf48213836db56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/75d69e3c55500934516243893ec0f31479d12df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/b4381c59d11b4347ede0486ab57ceb64731f2d30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/dc510a359d87bb17c67b7b981106fb9e5e724826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/f57e6a5b2fc4b087bbdf54f99a6120af86cd95d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/0019407330731ef16afb79fe7c8c38f76dd11faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/1c1ddeb062d036be7ff3bdd29b111893d21b4acc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/9358d8c40a8aa393aadff30b37f42cddf09dcce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/970874162c50251626ea78304902117bceeb2b75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/ea3056bad7cfb843335d8aa021efdc2ced30729a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/f2633b38ab1cd4be121d2bf7c66d32ddf0f54385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/02fdf71a3ea00d34b13017f61907bcd7c82aa919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/16cc22824c5df9e969e469133b855b2130ac1cd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/2bdff857cc6c75e2c108fec77af40901faa06aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/2bffb246618759663688b16c25df7946cc5ec83f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/9be84e79892f1e3441bb2c55189e37095b156057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/b6d8a948e1f5f7506670ca2c5cd43b3ac39b2729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/1b04008f83f1269cd949014716b8c99ce7546667.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/42979bd759b4b3ee0c22eb241e0cc7a18bf6e196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/67260385d2214d1dd5fbd083c75e878a3c4dcc75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/bfef6c2f9ab0ce6743bb216623aebe05aa162e34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/d282fa8977c81de9820b8da466e74dcaf8a455e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/dd3e6149632694482cab24dcdd4f6f16fcc779c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/eed4d11ee34f7e599efc072930e66fdae1b6f645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/125676e5a5f33c1a8588f147baf13a9521235f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/2409a22afd266f4a75d0ea89ef7cc2f6c3300126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/3a41ff0f787db92e152e2ca2bf5fc9a72ad33460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/c651c9e4942b0ef5862b9a4c169545ae6399d642.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/e82d4e9de88e3a6ce49aa4cf8e8f39166cabfebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/f3cb166475e7f831bcc6b5c3357f390aa1851276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/fbc94bcf6013404857b954e2e9cd1821872be343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/3ee45cfc333730c0bc7bbba29a82a2b789c23553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/49c36c486401c914c243a20ff7d8e10c69a7dfaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/b1518d6d609927003c7cfc00037304c45d7e2ec5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/b188d0484f2ae3f5d44f99237aca92adb67fd782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/c16bd1caa5a170b64e460ceea2d61f2355fb968b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/fccc632c7053a748745bd2006dfb1df014c2d116.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/2729e72420581750cdfbe38b5b5d518c40454520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/2de6b6db2b33199290105944f148bfb40f40ce5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/3f8b39509f41ccab377258ce46107f8b7bc4685a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/61328b9c9d765a10aa45c463ee6c687c5b53daea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/6a79b3612a5e214f83b5c69eed636a1c5e9a2d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/f9c34a661fda7b404e6dba006f05d3f52223b01f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/2a359b56bafccc565c5bd4ae898cb1ba0daafade.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/4ca5540114e11ded8f2c6061eee52c756d5e5a9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/56918fe084f25c64312d3f90a7d6f7227862caaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/650ac4f0d843975eda88bd2a704783ac2bab8bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/9e34bc0d05e8f4070f4704adf83a01740a4df705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/b0bd3b58a1c5c39a6c7f343187532a0ec5bad1f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/03adb75101f4b345aa9966e61feda824b2182006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/0a1abaeb63fee5c5092cc6b52ca9404996f5594c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/1072ab4889ffa10cdba5bc201f60cbf2d5705f8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/31b4e98bb4a7b2ea1e1d9f5cdeea845367e30528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/9f0d18cb857dc61d20014011abad08c6d10381f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/9f9297c03d81f00be46cbe120c0bedca16b25f6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/e3a3d414d9ac3d2be884afdae27a1183129fb973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/0d7b64cb5ddf22efea27211eb520c216afcd7b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/16b995ac56b24b4b1e5dcb9a7e4025657277f8bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/1d0cea56908f147c645ea2e980eb37e0924a6904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/a165590c1cc89e42998b8bcfaebcd8aab38ce090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/a8548ab26afbd68c8c576e7d5ab50947a37e4093.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/b97f15d2b898026d61ca54ef6e44c09b8a99ba09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/d0863851993760c5eb43f181f8ffa0f95efc16b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/08ccd02067785f7f86af97e465bf0d94b1d22709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/0d8887175cbd03147e777dae5a84886b65103291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/4de61500c630157422ef415eed012d85e2306dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/70fe0a003cbba07f66d6ea49db6210ce39add4e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/c2f6ea12044982aac03c1724fc2010ea260c0fb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/cb30cc0f00771a529b56dc8a57fbd8dc2ac98550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/35a84ec6659dfbcd5b388dd61420953f02cb0c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/7baa8b8450d95c487c3684fa6fd6d333d6d93bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/8b3af4fa76b70e61b02ad5e1b39360944b119afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/b21a6a11c6d40c8f948fe87b48a09dc157327a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/bfb40f453576caa17010eb9a040a89b5313f1726.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/e646894c430fe0db3f870f1311bfd1a4c0296f6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/3f3faf82fa9052d2f6c022f7f4e97ed20a49beea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/445614c22d43c10723ab7a1444074e52fa98d16e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/5ed3d3234fd38ae97b3eb52e196e7374fe0a1a7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/64a6d199a4a1fd20d6acaa90c58a7b97f8a7c2e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/85d209708b175fb3fd47ecfc12475ca46dd24e9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/f23c27bf1a1c01c3d59471ebbe9fc3f90c385a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/246bd020b39ac5fd72819fd6c7eef90e5896b24b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/53c4a400a601471efc70b1a86b29e4f21c9307d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/8668aadd30500d2cbf34cc60d7985236c3a1452a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/9aa7eb228df3cf3fb7df8de4e4eac314306389f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/a6ac6238d5cd794acf4fd27959bd77d088803f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/bf2ad689187daa2c68c08dfdff6752ab7ab5bfd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/c81141d38e8e8d35fc25909b8de21f054d8fc195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/15143916856a34c19a1df96b402dc4375faf40c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/1cd29d271083723d2ec8f2e5c572db2a8890bbc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/329e5a6c37ccbcf51a1ffe80891d6fb10d758f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/4b6b49104a0814406b2c763950f109030db8cc89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/891fc99a1e91d00308624f139a3e5d767a8b4fb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/eff7c73a99b7b138497145685b5ee70be2251f0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/fc85afbd8645090738f42ae05ad3b4f5941712a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/1c63f12c6ab737f1a9bb5a33b4f2d5433505d212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/1f642b20721a4f44d6b2488b900c669a504f32f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/5e7c81323fa2d811f2018e7b0946b3bb2808eea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/92c68295ab846351256ed4f9fd1f8d741df5dd00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/d35d378c2d609762f495016010c8a4bcd03dcfa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/ea70d5560dfdc2a927e90f0ed90e120b8e18910a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/164e71579e8c9ee21829c6d88c58a4a98408546f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/333d11b928e173c4776209ee8f65b086ef476e61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/406a293fca63eb898e6a62f4273ccfd7e81a56ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/bb27068fbaf8e9c6069f36706304531f9f7697aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/e87c59363f6a192b4bcf39e0b3d8d2768554c225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/fd90e14f8b865985cd51894a75b4b2eb805e1dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/535b9c5f2483279ae7cffdaf1fe9e5f4bfa218cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/5994dcb64e9236825612bf70db2faf751ecfcc72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/870baa042bf96358754d3546e974f1f55b68348d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/89afa8f35c1c99c60d8dc5249e5b7716714539bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/9999eb29d1f55567ca9a61e4d7e6654b0e4efebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/f3ee72cdc6769210e881aa002b13537d81c44ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/1cc77bbab9b537ee7afb3f0f8040215a70fb4b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/3315d853e0cdd326c70e574cc280b591d7c7008d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/9c124d440550b64743587ceef57fd26902e1d14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/ac418914d9166e7dc4e3ed73a35b9a79ad011a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/c3ff26cfe6c9d7bb8fbdcd0358a64b74ec6aa32e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/d41340c2536a77ada94ca534911729fd70086a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/d938a74c4e014bbcebf99c87345403ea6983fec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/454ae4dfe39ee5dd1396711deed2c4a08b5bf1f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/516d4efa40a79a235631558d6c60963cf9b81dc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/6817f88d49c33e7c1186738de9f41cfef43a94d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/762704b55ba295d12e1d64b8d97909a495bb45d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/7bc01c10cbdfc620ed38dbd3f948b8c45b08232c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/b44f624593ce4d39d032947c2260586f5c494512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/baf3cd5c622710f34417f2fc72269272c134342b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/0ed4bc099971c1885b6f77c537f702e5314d0e42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/33f62a2bc115aa7e51cf929731b82fd13319dda7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/445e4272c37d22018842e1321138ffff19db257c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/b2291ddb060beb380fb869fba8d0ef7d0da23673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/e54247105faf8720ddbfe324b0c0f5bead8bb6cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/f811214b9056674a97ede85433e2fd597ea96d48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/04cb96649b6e45a34520839fde3ceca8aa2d4480.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/3dc6e902929c8459cc2ade1346c6711cba2b936d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/42becd5805a112ef3b6b787be354b604b8dbb7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/5c9503ae9567a71b2d8f06802e89ae9494114b43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/69604da3283cdf5943422a6280290e385fe90dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/6c6b0c1160eb108fab219b3429927caf036e35ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/101b30a14becfacf09ae23fa3eae96af87491502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/209ce7e80c9b455c61d027007d14873c9774d4d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/9921fb36149496c48668d9a4a436bc868869af0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/ae404a301f7348b672e3e9a003af5ba9ef647296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/b86815f77c3f567a2f0284c0c4a75fe4a1a3ae19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/e2f37e16ad41b53f52ea9e00fa94aff318756d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/27070cf6ca3e8d1f03e540779170e5dca82416ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/3698d498eabd69bbf963fc80e0d0c23ae4185bc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/6a8f76f8bcbd0cfdb9acbe5ee01eb9d6ae2e6cf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/77e791608517030009c60bea57c7481391660a69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/880117900115b67c58e6559bef93e126c124c0d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/8be1cdc36ebdf9391909e44ab1a9219fa2591d45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/b60062b7ab6b45190f85eb2794f43fb3e95ad720.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/0579bcdfb1202418ea752ab0ca14fc99599ff163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/1d7fcd12109a430d9ea18679081deeac21a9e561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/3fbcec32d1df8da0eaeefabaf4ff1304d72e66e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/4513ba6ba6aabe01e75c510251c7fa8c076fa4d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/85c73d9a7292fe6dfc6b0a78b6d3c2ea1d3a4504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/dd861077ea8febfa1e3acf27cc55d5938d96dc6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/dfde475d70c370f1b9e6b4a3b1c9232a7c840669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/3168fe192bb7dd1d175a969b1529f516ae830063.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/3f5ba844dfa51bda987122061d4231a271edcc0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/3f970df363b2d3a5cb5d65520e67198d3fcf16df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/63a4eda6be4007413b1f78deffaaf8dc0600ca96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/da916d811a05034f5e1995c2deed5d448860fc81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/f38be1719f5e79a1a4ad3e553a741f64aa33efa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/11958a9474e5be76099ac4bd7484855dadf63c40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/30e43c2d72c4d5b8d715f59e71d95b1beb304b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/435f875862b13823ee145842a36e3475fc8f72dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/6467dccac6c2b5155685239bc172e2b3df231017.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/c8eac290ae00b980b7355ca4f4e082ec3c5d5d48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/ce57e01889b96338937ede10db1e89d56c15c3a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/3aee5c69019fb95febe3ddb65accc4d4272f2e77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/4d57e683fa970188e18a9eb96721424befd7672f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/b6c5efc0c46996ba0e432a5f34f0715c9194d105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/ce72778c166db568e22b216b5abe980fd8613d72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/ebca113b33e109747b16051b5727a18b15e30ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/fc8dc60ca7295cc0a1dd8651d43055e33c5e94f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/3e0fcbf1a79080e25e83ff44c17c8322e08ffbaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/5d0f3b6cf1db6178082f353caf41a0d6d1a92d61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/76ce9315e1a52695cd3e117740bcf16c5f4a25f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/a308ac05336b37d33136dd2561015d81246047f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/cf07d8290436c961f3cd968e45eb03822dad6dba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/f52df3354490ff9dc471b68192af2b02ef2b8dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/fa743878f4b86f878d93134251f6b3c46006d53f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/36485ed34818ce5e68c27e1a94a46e376e39ff51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/3fd0fe88dfe4d2f8d3b084567544337a1e998ddd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/533dc606a262095950221522099729fda98c492d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/7addd4dd4fe27e8bb45fe7e418a84239225f9f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/8d8a4aa0a3b8e9e5516dc4a3885571d5ef2f75f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/9c95394b794974177a611470d65a2437174bbd27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/dbca028e56adcd256e49164a5b2c7fdcca4588d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/15f66b5f691adc6a8043e05a4d084b8a0d2fbfcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/45bfce31db77278a14c721aea2e6adb7fd83e340.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/77f2bf0831fb7ae1b6597f6f4ebcb15435b2dd02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/a499b4a36e3b56be3f95ea614ba9a60c1848553c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/de2075f81b9275c46fbb7f6168b492a95ba2616d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/e26b4856e4c3779bab2c25fc06dc217a47719101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/037f4e81c6d53a3fbec732d00d6c9c0cdc34a43b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/15afe7c33eb0fb70c62dc99283fb21aa95a3ffd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/4a34894448cb5c2d5e93f5b7ad2349ccf874f4ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/5135e902a65e8d939cc388dc817466a362fb4749.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/bae20062bba6f2859900361f64ab598b849d9182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/dfeb3028ba541cc7a8727ec67f3251121f11f809.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/3d553f9c018ade5d3c6022ece6252e7c70abcc68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/49b9d3ac0ce5c473d065706fe9771361882282ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/4a8c9b6d228a2a5bcb15f929697c6fff5a6d2d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/d30675987a659105adb6406eec3468a658af4f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/db3ffbd4da741cb7009bb3dc79b2039b50bd8526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/ec88d6f15f06d6fede6c5276b6a472fe5c57c9dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/452d45f065259b312ec687ea379688bae1653a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/5be448ae7d5e6e4444eb62c918720187abd2a20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/6b60aad3e089b1c76f8b5c21a37d47ce8e361b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/87a87fb989dd711837af3618dc94da6afaee8a23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/9c1a1c82d852c3483f7b18b881057cfd74164852.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/d15f82002daff025353c6542ff81feddc92185c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/feedba8f90ac15150fd94986810fb84413148e24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/213d6f3ab41927b1ec2a6d1186b698ae3faf824f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/2207e9383ab1c8649af6d44b0bd3dedcaf47c6fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/31006970bd7528235f30e3caa04b774413006228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/3b4625eb20b79af81fa887b934049224e4e7904a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/d45fe02dfc505c91cc2ac3ca3f49b30708b98865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/e25421aaf4f06220d4169f8b91a7577e9155e3f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/f1aa79071fdc8a0528351548b4489c6c8b8de254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/0f7582300ca091293fb29073c1d79493306e0bd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/13fd5568a678f17912d975e52e378b5463d97d31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/1d487cc8265cbfb281b9ba8ef676d51d9a5d5335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/3770b7461b5d65688123be45a6e67df671912f7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/dbb5600be0020263e7d3f5fe944a326c75a208ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/f2b2e6b7618db6b4db08c88e534e22efbf8c2177.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/29ec15f8e3854cbeb1d8c31d418284128211a8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/475bfd2ccdb402e9f1b80bc2851e9382964795f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/5622e6ab42cc72c6d05d60eaac4f293e6c5e81e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/609465ce4125357069adc1cc23189f99ee9dbdd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/93a3057d75927d200968d49587809e3b441aeeb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/9f937be7477a777f9056a71a4a99b7b7df374d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/38b1e97326a389b79b1fcf58539cab885332f59a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/454400857553b8de31598e843f0762913fd8ae7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/49c59e1f67da143722f0d54ec0d75c625a51589a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/8e1de8b39b6da313f0f0de11ff5118cd3b6dc8ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/9f31007a8603dac0725c999cae45ec5387143684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/d1d83405617b46eb7fdb966cac448f3c8bcdcdd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/08ceac0e3f83b6924a7d15213299231005931111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/2f95cde6dd42d4fcf25f0661a4a55ef701d2c486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/4d1c6dcc70b44d0574296960a1976425fa02bd17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/4ebadae52d4f780ec335c42fb1309813265fbc8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/6057f0aec2fe61d73f342b8855098a45b89a7ca2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/7aeef82a445101cc4a5ed0d1053802a7052c9fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/ac745054207fb061cf457fa5e1a3f261dab5230f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/085bbd247597da7c823d56bfb82d801adac55367.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/0d74b6b5638025accb7895b2ee7cb8856fbcc736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/77ed1b450895cde1c0a8a609dbe31e2a5d61b832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/7f03e16f838688c03e0ec6b3651feb8217a9e47e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/9e0de0c90750c64f7d51025849ef38e79daa4c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/cc97b6e5047c5be9d0246d3c273237502ab1efa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/f2716f39ae7aa4f4d0563b916e2570764e44648e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/17c1d4993ca6cddf8d11601c38bd648309405437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/71b5f284c1dda6d35699b9b02153d75e72215952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/81964014e4571a5cbc0b41503242cc546e029d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/8a9ffe7a795b28e56e1224600c92a0834f115845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/954ceca5dc201df1916887fe3866725002eeda62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/db97538f550057dc5b4432bc8a8b88236251ba36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/33a5ffd3a09d1d2159daef06f06c20f00503902e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/3b8a8dbaed805b4a350cd8fe0e97d0e093ee4325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/aa24f9dea54fe86d9e6adb7a45e77c43621cbc1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/b2319769241161b4c09d0bc3bf4f1f48596db1e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/b9689b86f8bc2b88b9105ab5c2150bbc265ea3fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/f6e31621f58a0b3a5f7ba02eba062548c4d0d718.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/1c9b3c46b481be4d96e20af1b60238dde96a767b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/93388002a9bd84b69136c90ca730920c5745ee17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/ad86fcc1f1313b7f79c6bba36fb6f56b2839ce7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/b4bde06c036963869d84b1f7d5e548638bb227f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/d83203c585c916106efb4bf188a4169b916ad4d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/e5958af5abc8a740ac20549730bf42a67fb75693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/269c709082d106e68b6973883967139ac60eb3fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/2c251124f58b81d618a8d0ac61f35d27c1192673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/39d7dbd3cbb5d1a3f76c4c0055e9b74816c0cfca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/66005d672cd90fc4928cefd4ed60c68b9ac5a57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/708ef09c0bfa474d540a525fd97f12de709d1b1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/718ba14b2a5f40e0a9be40c58d31a066f59ebfa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/7cf37e6a3c853ebecb99a8126f09708a48b1e92d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/073cb3c186706e953655eaaf9afa2e17b6141b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/191db966e25fc93bd16cbd354662bde0f6602573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/a521208887daa78bf381d23fbdf608fe73479f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/a52320680dc72d3a65b3861149e7c04756bd6410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/b16773325384c4892fe1a647f49ee137347d6143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/d1c5b26c77cc7d6fa337638ac62c4fab00af5aae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/d2910c0877807ecd69977a127885c49b301e3287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/3404e940f0680a05ce1015134139cc6c6e851d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/4a9e1c387d0e60d558ee114ef6bce7529afccd5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/b57e91c4809446726566efbc446200cad099eb96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/df110dd0fe376340d977cfa4988d7073334d36d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/eed0ace4a5d4a3f87e7c4b30cd5fba7527c99f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/f880994b823d0a323d0a50a20cb0b79abb5c4808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/12d0b640342354b3f9afd353fe8b1dbef5534e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/285428b0ad63700defc2da3b8e7b37004d9f453c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/4349537c354cb19635b4fcd7e04131b23cd578d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/a4f4d7323e25b691a3a9b1832f650c7b5b558bee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/a98198302f546cc5fe5829dc2904724e2bcd79cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/c276802110c00a4084d791c4ca4ec5215c0ed873.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/07e1ebaf60540b5cacfecab42f35b6ef0d45b140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/54c2adcf7f969439d80967b3ba8209a50a466cb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/7dd4ac024fc82c4e58a3e7599ca0902135b718b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/7fbfc4b3f3f8a9b9f119036d2585c4b05ddadde1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/80ca28878c02f4393bc24c1547c3c3776c164086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/f3f30155f331a5ef5f761e3b4e875d3aa98516e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/1633d04623b4ea7cc058d5b2fb7335a037280ed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/184fe8c5e11b52523e3ef6fe392f08b6066b9c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/7212d82114fac91a9cbbf4ce3005503e7bee7623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/7a819e1a2c9ae434f0b96e8b97a2411926805637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/a3456facaaeab57fa70da02a126962c2d8eed13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/e912b2a98e598c514a11b387cef3c23202e349f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/f47f11bfea6ac9b892ed7318e7581bf86e35f406.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/4e47377686af0992904476a615a65764c98e68f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/7029f4f9bbc964b95a161a3449ec9f33b58f54fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/87e9f96eb700704c0eae89016244e6d0c0551c69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/9861977212ff9520b22ea46cb2d193d2ee38d542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/b9e0b588ced83c436e3b0ed43b77f8dee75b5e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/c1b7ab37558dafe82f769d9466a81709cbc4f6ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/fe34e1e359ea44b16bcdfee3e03e9abc3a56c91e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/05d7d83c4f0528d93e087131535005399ed2c87e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/0e89780f7c872527ccd1f8ea5bf065c36851c2e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/1d8f2e731227ec15d5a9228f704f0bf764995e7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/93b3577b0c4cd41e56a55a01827736e2d8c85e2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/c7031de8c78cfbff0d0f5c19c31f93397c55058a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/f0fed9a21bf9553e78b8e3f99f3269558140d99a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/08d313d59186a2e8d8340d4fb8355a467a52b273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/530cb0e34f7b906ae646512f0863f36026829007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/845dd1fedb8cc342dfbef6e82dc4fc65208292c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/c47275595071877ad693669845e5064c07703f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/c6aef87a0402b6b1da20ea7407ab9d34303f2ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/ea5806ccba05a270d6ee181e03802f751a1ef7d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/150551c219cd6eb68091b773c761d6e4a61f9bd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/2a1ba4e4ae50f3c53e93b9053b30ecedd6fd214d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/35b6234ef171b17980bcf0dc7b4c5e4af106f9ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/60139d51ebb0109b5a75630c4c4dcb42096283a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/a33bcabff73b29dc036a625b82cfe9492d18ec2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/da028d0fd3e92bf33263334c2c899fdc4b74d428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/0e48c6c1e0ef6dda211082f7f3398a7a845691c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/0ff3deee580feba2dccfe604ecf6f68beab30673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/3ecbdd30becf5af2f156fadb3ecea1c5eb39bd70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/422812be562857eb3c69cee2f25d908915f9d510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/a26d061cd0f3be13013cdb53dc8778489290a4f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/a4784e0b47d9abca69d1bc7ef4c8ef322f8d1588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/d8dafa6e459f1a95acfedef50cb62896a1ebc022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/1a12258177541e114ec754da30ce4b757cbf85bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/29a5fc3a4ccd33a45da7b6dd583c9483580fad7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/40eb20757334190952e394ef07fe1409523c9ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/4a0f7566b93b76ea58839c6775b83699f1709f5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/8192bf3b132c68da3463e45823b5da34264470e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/d30aa88feede6a443e22b806f41f625d32625393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/ef94e575d01d58cebbdbd823233838daad429b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/2da106460d699e960d2bde0a5ae40070182390a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/4c402d7980458055ddbd57f9c20839c899b287b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/5eeb1ff5a09059996bc8cb2ef0c85ed1f9d69a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/7c55c48da845e72e138cde75f7b32be34a31450b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/9ee7f8b69977dc6c708a177e891dbd42577236c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/a3131fbf3ff78854d871310ffe1eea6f0090bbc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/2e6862b7c8231650f158fa8a735852a2c65b2d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/4041333021784e459f3cee06a0cd4bfce6de8317.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/690cce97a580b67f0a5c7fa6c3f18581d599f945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/6c43c7a9a914912b9e92222717eeecf8803c03fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/a51294a2916e9e94d9e226b30bc963e36fee2827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/b485f244a9c58a578ce7852c4c3c4f424ea14364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/2a825b5c58380392840679e1ab97d1adee472afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/5a7f9ab18b6b1e0048f73d00a9ae30b5059ac66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/c245d46541f55bc1ae17ad35b729cab4fc348dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/d2ee560650db3ffa813c25c44760334c58422fd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/d7f7e378bda090fc2e610fe8d3d4b66678e5f191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/f8c84b05c1f85be9e7b098d072e2225d3ca606c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/23c4fe7bd0640d7ed83017ec4c4d971c53b682b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/47105bb9568c8f31f13af6913baacf436a66b94f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/6570fa068a6e6c3abbb4c9e69cd23d50233eaaea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/8f6a10c52af23afab0aad08441b23e5d3bee98e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/a01a17e4aad91a055e6756e1eada42b06a9aff73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/ded67b73c15c23d71ba301cf37c68d0f86922e8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/eedf3fbab6a61c74ac7bc45b6f76e7ac6feb1535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/499e28731ec4e29b816e5fa004148d61d976437b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/6b10ad52347399325d3a8b957bb0d4dbc5aa2492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/7b61342f15887414fa2b816fa27663e876a20437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/96f47fccc308c8b8754486e2f1e928e2fd9bf864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/c272a50b3a2a570701d0a05dc6538e3fe157e549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/d3030773aad5be96b5d70da97b31fc886e286300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/f197986ce87610726943045b00b392e6b33b58e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/038bd5e3535a568ec15900cf9ec51e20a4e5c1c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/324b9700b49dc1b9f9365cf17577756a83193bce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/48ecc0e74f3913e8ab30dc4281a62df9b7cda859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/c49152a249a033e687d49ffbd36d827790c60c78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/c96571b3d14c3521f69c0bba056b2a2e3b5b357d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/d8d3f69cbc1c078d1909f06b57a3c42fd8ef0adf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/0b1e10622f21a769ef1757d4497c11ff2c67963d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/6c6b7751ddd4481a0f490244af0adca7fed9416f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/a57c68ece9dca1cdfb90c3b5e73a00682904c2f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/e9fbd6771f6f137287ab6d7a786ba53712bab4f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/edf465d649229724f4597c39f4c2ece6cad6bdff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/f5e0e5f7155683d3d5e4d3a7cfeccb87c82a8005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/03c8f57579ed676fea9ae176b915fb69e90479b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/37e30bd6fe20922e70baf40205ca6c8effc41d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/43c1eee8bb6cd7545047bf793dcc2498e9a56d66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/605d48f779bc00157a5110faf4e55759d9aefc49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/88f9128ce91600bfb7fef4669d2a8de941a4c11c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/d46d1e9be69fc0268e0a97ed40613c0567760b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/1f8f4335c81412044d9f5c8b0fa2948f4d2fc3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/3add7c18cd82484be5c6f19392e4a042516a877f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/42e684359a0355619c7ea89ff07bc15705995f17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/4438775debc1c5772ea584cfc3a87dff9c8b30c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/6d100dd9037cb767cdeb03e2498948d1c29e72eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/7240a4959e81d25620e81e36968c294723e9a096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/b7237391a08ace2dbb30b7212ffcea3ce9b06d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/07c1bbf5135ff81c7c60065b5d9904ee951c7f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/341e36ee0654b18aa6fd8e54ca006f6bb3e27172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/355a71748a5ada74cc30f8bccb56cb4058e54501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/4a94e28c57677ad8864c359b1d489027007aa7e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/6cc691cdfabcb31223ce850748d8dbd0060401a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/cb80a94140f2eab4d461ea35aff03505443ce598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/f9f041ede0c5583e6436e45caebcbfa3ba07f2bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/2e00c4deead135a1105305cae7e820d297f2ce2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/38ef4b309945c339e33f1611879bd74ccc5ca63d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/477b81de9d66f92c42fea15f47f00d94999c8d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/4f08a1bf2e202c289ad93b00d52f3a821fb2ef90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/58d5555eb85e6c03f3f892b065aeefb333512c35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/80c7dad860bfa037e98512f24f2572f68842a6fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/01e348e96dde7cf22429feca3f3a5246f55d0c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/2a1b956e89f41c7e04895652696038a21db8ae42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/878cd71720acbe6c7bf467fef8ef844cc456532c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/d7df291827289bd034712ee1b6d192780d551b21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/da9f0fbb3adcafacc53a45d3e9bbfc04ea604022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/e4e233b59773785d64f2048dcbf7f5f7876e4614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/0585bef2f62c9dd22410751d2a281805d1141c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/2cfa3988f8c451748fc6b59559312cc703f5be68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/425171808e53999c56911427f7188b1c22da7983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/4b6ffbb89e2c09a608cfcbd8b36afbef1fc9fc1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/52eb6e5a9ecdf04184829924dcaeb6adddbe2887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/649bafd4f12022e0858850d5328f12fff399eef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/0ad3aa0f39d45c6e08b036a507ecdf3d90f05ef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/0ff75cd33bb5daf3eb99465ecd72eb3fbadb53ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/78907e1d896132e5007a8b994a51f25e0cbf4f56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/b0511a387e4ba7ee1625a051295f72fe723aa910.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/bddcb27faa6d2a2d27c80bc4e907a152cadcaffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/cc80be7b2031a295ac7d2919459286f04a5807a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/f4abf32103e98c5adf7a47b85070b7209403ffe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/01b6ef1d6c651fa551127873a223e3e3c70b7ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/1864e497742e778bd26072ae93fcfadeaac2fc25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/7c7a5f704021faa0fbe65699b5c83fc65607b4d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/a7eb29d1511d8ea6236b0200ba5915a767206a8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/eb22d5ed81e11557e778327de488a6ed8a70722e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/f61786bfb5709afc929b42d65dcf2737dd84768f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/fd97907afea58374b382be101076c5cc08d143aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/53ae84ed52e09df40a40e844e19c8d5888198309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/6bb84070759edd19b45c3642c1dd89d74545fd35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/8cbc855d468b45d514369166efbf5171b346afd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/8f6ae004d81be04e3181734c3905916166a0e184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/a8ef4a621532760f4d27410eea935360ace53671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/c6b222df2ed0a30c779c2559ef6b97b8362204a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/3e433bc3662e1b8c1a948f9d45bc8d61cd6b5538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/b59457945d2e331969452baf370ffd710f0d89c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/c18d685eaa7c0777c03112673ed8d5b3afdc01d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/d5eeba62431529bb01fc46aec985b78883910e8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/d87408c73c49feba9621cdd53ca788a5bf0387c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/e1435c18d6f24f7f88c1d5f627decd6ed01927bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/1045543f0716801a8ba9ab027beebbda84c7744c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/28e5fc52e6061f15b1fdcfb79f58fb07570f5eb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/30e898764628f66eb358676c928a2e435741d861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/35b40173ba9287207b5c995a8244681532fde913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/6b0369a772e58cb6cddda68b9df0fbf347c932bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/93566f38a72cee1da7ee1692fbe6ec73bbb39959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/52263f8068bad5389e6ec12680f94a26e250d7b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/55e0d6446b220e60b3840d90fa06209cf085e393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/8c1f8b7c94d47a91d94f9df01ed56be0eb721962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/9fc09644cda0c7fa72a3363d22ecb332c58fdad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/c94f22cdee11d0f742d11a5a09355c5e89d78098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/d3377fe6bfdc78524bcd99f11aa2953c04a221ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/fe4136a9fee113611152905e0c496f8f34650fcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/02eb7ee2b6546e7e3bfdfbce0a7dd45119a8e0e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/04777b2be1d3ee2764e562c246c85dfd3dca9454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/35c6a0af14bc1a2c9ee982187ad03a755fbc0301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/939513497c6c207d3c364f2a8da819ebb5ac1b84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/9ff4402fa02f46b8525909d5f035abe99e0f6d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/a8a6f0ae3ea50b5cc9d1e7076f7843c106171622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/d1c6bef67a250274b9f6e8aa82cdf4b6baf4a858.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/1ea8b1cf2b44952862b5bedbf0299931199845d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/522bbcc99ba3d46995e0bd6b6a0f0f1accdb4c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/918b108738c54d8a4d7adefaf0e062b2efa0eb96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/c168f694b434f4c9b8834674b0206e5a696da6d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/d6fbd492e4fe0653548d89955dd4158a0f856005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/ea0faaa9321c6852174069135abc61fc039b83a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/2945cd6c4b1cbc546c03383458051d03362da218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/2b8112eb2384b84da5c107fed01289af864cf42e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/59ae8dc9413e4f4306720d61c7f5ed456afcec19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/935943e1897d4ced5e310d90acd8cd79f9a06c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/da4e9624d3e5a0db7e68d6a572735d1ef3ef1c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/e5d752324ac78a21a4d3577a262184f6fd256d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/3c770ed1de02623693954de0d88bfda4414cb218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/3e0d94502e80065b849079cedf77c984abc30b2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/495dc314418989d7ba0dc8ff259551abf1cbecf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/a94c21df8f66c3b7a68577a566479f9f6f6ab437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/b109a0dd0494fa290cca64e61cfbbc7d48afccdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/f6cb02d61ce8cf0292408f92a18a7be2c1b77a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/0245fed9a9b5692a0c8ba722168c82dd6b7cebb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/0e5f45b7059d66d368bd09110e491b786acbacb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/5b7a79767fdba7134cf3c7a322cef28a1b40b799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/814211f64fc9851564adb858f4d9a3e10b420817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/9f0b8d2a01b1dc49a501edb982697ef4379ae8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/c383b5699e85203d770e01aa7c9740a178566df5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/da5826e880417e1dca747a642bc6e5bdc31df9d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/13e9aef79a3b4e47345db107d6ffb5f9eb47bf27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/2dda9c44bc4499baa53c68af6823295844b4d2c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/3b317afbcd25f2cf1be9b84ee6ba65a24ee926d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/55008443e7e872e8091b21528cbb0918b81ff886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/60c44fce7b8c69936f14b9d152bb761a4ed6b3f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/b20c4cd3b3b9bca26acd2635867ff0a2c0dea4c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/e4f901dd398b9da800ccb2280fbb148db14bdf77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/ecab17b84ef4c8d77085f1421c36101ef5bbdd8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1b80f680433374ab54ffe64288c519f89135cc12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1c286e9a98ad5a4463b4b06beb0d053524dfe353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1c81404d2ff122ecdec55b755856f5b3d229cd6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/21dceb79c6a4072692919e4de16a625b4dfba444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/29af643f9c664e60f84b9cedb26904e224521a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/31e64dd39a87b672b2eff31fcb8e64e01f83c2b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/320e1d5db6214f298cce7fea910c672a8d45d092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/32a81a909b33e6db38b5da283472e80e1fbd834c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/3386410affd360f79cf42d7bf9d76906496548e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/33f23f5ca2eeaad59f86213ac83f176266e4fe5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/3565139af0ef633dad63abeabb6a69de9507e1ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/36273cab53ebbf062867fa2f71ef892c4b676553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/369af29bee40ba2bed686b0ccbc2b5fdfb23bdb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/39aa1b0bb08d5527d01afe5d1df3230693c974d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/415b5b0064038d39ec4672448a6332395bb59814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/4639d9616d2f5ca29d1f0167604bb417d65e0120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/4f6fb487e7c5eb223a0fe2f5ef9c2f1ec1cdb49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/50da67177391718a0f137e9eb7621dc32fdea3a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/5208236235625f67812a3644ac1a33eb7169a61c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/53d2db6d2e8d92e8081f1b014a77e11258cb959e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/54db144f9ff42a1a6e9c8969d41de0f035b5cc0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/584277145f2fc25748bb5a374c5cf57bf68d1913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/59975c0a6e15775eae24c6e4e87a4e0f9d46b039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/5b56cc2285d54ce02856dee595bc4d1b84b76633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/5eb2dfc127212ab4b0edfe105c609e14e2e111d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/60a1df6bfe4a7ce3ebd7b899b1d3f2a6876e87a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/656d6b9dcfafb5bdc8f20b0c65ea64f08a4106a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/6fe16d9765bc17d1c74c4bf2308ca8762a034900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/7677d127c09fc08c19739e4b848967bda02dfcaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/7756056dce492bdc2cec9354f0f8f8e398d98ed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/7d680fc60515c0328fbf1d7fd7eb7053edff1745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/84e381e6e8d577ba686d041bbcfaf02784518980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/853a81b3835ff60fa55d3b47e5247fa89993458c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/85855961322e63e566d9b485d3afdf1fc1959c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/8e82fa1b5ad3317ed08fe4a78a0d00be881cbf36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/918bc1a84eb8e0da66c275a7de2d4b7e478041bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/96d715c1bdbc4f6b9f340688d8318dc34d6724c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/9f18e205a7138d66cf8faeb2428fe1fce9f11232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/a07eb0c72d644193b5ae717778b9866c8e52de6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/a851e6c67fafa6fc780e1a08db675766a18a8756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/a94cf23591fb0f860f29d1d722662dfbaef44c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/b3e9e2bdebdf0d1e119aac69bd3d0a6eefd59e14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/b812197b5f92c9873a9cbdce5a7949df4933aa27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/b9a65855647d616b95619f1300d9c4de7b440ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/b9cb8ede256da9e9e4cc812ffdd88524d943099f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/bc5d0a099a90af028a49a6e708b5699772d605a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/bef4163ff0bb6809c6d54020a5c89d368a094346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/bf1a5cd0353c34139d2875fd217d583c83933d88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/bfd653f05c1f250afefcbf8cd8c0e89aa14ddef6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/c55fc6960a6268e02503d86e62c5cb6b17a5ebe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/c833756daae1cd46fa09c64ef9742f153d162ce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/c90b58c7f019f10106ec1275f92c5ae2abf3ea25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/cc426936f426919dd2490792dc0869403dffdeb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/cc6dde1f97f55a486c977ac12605a9fe106e63ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/cd3feaf6b2997a388bec48a76c454b0749277e24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/cef4860ed0d50368fc511a76a9dd58ca6aa54a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/d37acbcbad840f42a320b0cff8e6bfa81dfd0e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/d4e333056d970fc95a21fe7e1a63af8876775a8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/d8744e39e334c216ce9f82de09f84fab17b1da54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/d906033e09f39145d99fcf06c49733d89e57abff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/dcfe289518132476ac1676a877eae8a6de0255f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/dd67508de8edb56f168825f56498324783b21606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e132abf3aec261783c351f15f563d30d9049b183.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e3e78a5781529f5ce0551ace25c568ea2de27c63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e78c324730609aea98d1616c43adb44afc60cfba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e7ad174f9fc19f4a8df8ef7d8d0e39ef6e82f0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e8511560751c0231c2216134a8be085fa6645b4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e8d18f03f17f3d30a1c0bf3a1d287c023555ad71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e8e5812b6a3549011c1d203c99dfaa59e2da4df0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/f408a46ab91a230dae4471ecf25c1739e926a6f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/f71b89df6607c89e33ee575e8edf657b8a4a3970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/f92ea1625c842a397b9f9e1fe50ef8dd8dd3f5df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/f9ea765edc793d3a2f4f64b8bd3db3fa5b7041b9.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2018-34/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2018-34/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2018-34/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2018-47.yaml b/configs/cccc/dedupe-url/CC-MAIN-2018-47.yaml new file mode 100644 index 00000000..0a379721 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2018-47.yaml @@ -0,0 +1,581 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/0c2cda2f53bbcc3daf91ef8a0d3caf8545eb7b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/0f9e7db85513bf476993a5ae44ba8d9c495cba92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/115ab5c9463c991ff1b4bec4a446a5b52f75a194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/122d3e400632716202bee22b49834eb65f0f5b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/140fa24dbbde3a1c24688229eb96f7e36a3c0d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/14f133dd848135e74ef335d21f3c08c42e23d778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741016.16/warc/094f06e908246db2475088014d6a2a8ffa3cc9b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741016.16/warc/534bffd236ec4fc2b6ff315ea2d26f61452ce846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741016.16/warc/8bc30046aa0375a11da4edeb8616fcad9ea046ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741016.16/warc/dd4313f63a291a42026dc494afb3ac3b8c2c5a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741016.16/warc/ede5bfaf18b830405d1e2215379ce7d9d96cef7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741087.23/warc/0455e428776681753b8fab83f502454f482be9d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741087.23/warc/40344d331885b5b5069c409ff2e00e31fb6d0e7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741087.23/warc/9413282fed941aa9cdefc66f849b7bb1dca9e803.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741087.23/warc/e50a855d3155c9b3455cb9213f6417220db206b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741087.23/warc/f2a3ed122197b67b11c2f469c8df9fae503a921d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741151.56/warc/607125867388c60960326db783e87ee1df4ab5e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741151.56/warc/76ec07d9e24fcca8ddc0301dd5598331eab01e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741151.56/warc/7c4069dd3f226ae191b86ff9ebb7ac725e8dc771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741151.56/warc/9135cefbaf57a5c0ee4ecf88a0e8551b216fba44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741176.4/warc/5b98d9478f273fc0ae934a8336aa5298fc7ecaad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741176.4/warc/8185fce0c13c6d63fa688b93d3f46dc90dcef954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741176.4/warc/cb5547c4cd4fb0aecdc55745b6d39c79e93eac5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741176.4/warc/e215a1643ea8bd37ad81d342c3db3bf840f9d3b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741176.4/warc/e70457ab27eee8c1ed61fcf479e894070b4bc9f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741192.34/warc/069950d81cc9ceeeeea81fa7bcf8511dde646f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741192.34/warc/4b9ab9b0c0a67faeff25bdc869f31339903c1195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741192.34/warc/84b4246176c7b4ee5167f8b3dbc56e16bb07ab7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741192.34/warc/971f208c76d25c0f7f3cbc24281a8fea4505e9e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741192.34/warc/9bbd042bd0079d8798547e32d47e3b482db40292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741219.9/warc/2416e1ce9df2a902230235c0fa1f87a770d9d838.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741219.9/warc/26ed6f9c82ec6338023a08ec2054504feaea8baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741219.9/warc/8bd890948ec6f85d4bba8b3ce2edb00a19dce080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741219.9/warc/8f7474362bc21ddd10a7a06208b4dc7adae1e706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741219.9/warc/baf6615e6abd7593189853ee9109a6112dfe0adb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741294.2/warc/58b613871ff281335ddd5a207fb2412290e80f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741294.2/warc/738c7f2156bb8cbd65d359711897b41c9e6f5a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741294.2/warc/8fbd812bea663ec9aac02e3a6f84485b3421b898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741294.2/warc/a73c806fcf1c653c46ea338e88af53d6212962b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741294.2/warc/c28c26f918a84a684f297875eba068351167c63b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741324.15/warc/0f44464d9615ea254b0500b9946890037bd0c09b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741324.15/warc/268f0bd42c3218515711b13c1a885fdefa9c2c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741324.15/warc/516c860b9773aafbd53c32ac0677488dfd67b802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741324.15/warc/d615e3f0337112d2daf8fa9962e4320c8ff2220d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741340.11/warc/0803bd4e8eaed22309ad353484e19ae54ca8cfb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741340.11/warc/2a0489774fc2e4f6833cadfe94430a92d0aeec7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741340.11/warc/37a8540ef1b98b04adce233bc01436cdcf373505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741340.11/warc/3f7bd19e785fb656b078b48cb89ff63e66d5e947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741340.11/warc/8e4a3fa147cb6327c858c17082360b78bed782e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741491.47/warc/378825c0e32e95b626d1162b51154a1cce82c77b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741491.47/warc/4dda19e593bd0c844467b92327af708cc5cdc768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741491.47/warc/6f3e08d2a1e61f88923e2b045d6624aeaa0a6d22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741491.47/warc/933345edd5b186f8930a68c06fc695ff25cc8a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741491.47/warc/986b5017c6f9ae638f7526e8bb3fa84111de4542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741510.42/warc/0ee8f15e267edec97bc5f40413b894fe781daf36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741510.42/warc/4838fd39425411211ecb0d2369d8db2a2600c0a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741510.42/warc/941ff111d78f0cae582d999d041462cd57cfbfa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741510.42/warc/c425833bc29cdf272a9bf70a8e7fbe5fe1d478af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741510.42/warc/c690dab09d06625b31522d1c2b1fe95774e3b6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741569.29/warc/2eaf74c2774a391186ce68bb4bbca1e122c8af01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741569.29/warc/86dae36c0593480d0618ec77e097438578aa7e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741569.29/warc/ed68a0e7f67bbd1d72f3c70abc9976214fd12068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741569.29/warc/ef37b719df59162f1b0c3237b17ce92f4e5efbd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741569.29/warc/f2ebee32dae1d655c5764221dc5ab6424f8431f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741578.24/warc/52cc96b4f165bc71f8fad8dec852945ce8bf0b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741578.24/warc/5f3132eb42fd0fe12554285ca8fd9f1f496510ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741578.24/warc/733721de72890bf2168795b096de208928a18c6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741578.24/warc/86564a97d581fb98867abbd26db55d06da1eb325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741628.8/warc/5da644c912f57da92187469afa028da5621291bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741628.8/warc/668f4256aba02ccf4738af388f787506b547d5a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741628.8/warc/ac17f08b214edc7c373d5d319f5a29ed11ba07c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741628.8/warc/b255b1f716368bd52aeadadbe0c422d50d17ee47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741628.8/warc/e064ce281a394a1aaaf046a87e312763f004fa9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741660.40/warc/2c81151097a6186076aade5ca119a68bb8305d2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741660.40/warc/532bf423d9bd732e6fe8d9d7ba388939e74dac1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741660.40/warc/c607b4fefbf369adc8dcc64b007401624b05fdcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741660.40/warc/e209984140d864b31914b57fc1148f38b9d87ef5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741660.40/warc/e92937f5d569d34175c295e41302889fffb5e039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741764.35/warc/3b1955ab4dca9709e5a803fd3f8766ce743b3b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741764.35/warc/6b9725b978fa58d51b26930b08b45150077f56a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741764.35/warc/7984768d8453eead2777989fd286ea3f01d72228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741764.35/warc/d8ce899ab23a656163d1f833bf02f7c0df740d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741764.35/warc/f85a68a225a53b417c04ae6c2941d3039b1462f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741979.10/warc/1c8aadd4240347011e762cf42fcc4d3878aa2519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741979.10/warc/222a82db36e4fb635e3441c8b9e281503de993b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741979.10/warc/5007181acccb340443f0f1b0b4ae94de043880ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741979.10/warc/da5586d9c7ed847c54f80e6fb42a5c498f519a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741979.10/warc/edb5d381e684a3a7a6eb580c01ecbeb54b236b3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742020.26/warc/79fa5fb6a02e50a1894d1f84cf4d4d211fb10e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742020.26/warc/a0bf872d0394df2f6e192fe7d75c00317c5e2ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742020.26/warc/be2254553f1236e478c35d5e641c0d29c50a6e66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742020.26/warc/dfcace55908835f220be60c3f0cd6f0640e01426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742117.38/warc/128de272125fe9422e517bb2cb1b83ce2bcfe95e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742117.38/warc/2546e7344fb0053efbe56e7cd75528205c61b5a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742117.38/warc/898ee6a581fe9dad691c7ad2fb3e8f2d60eccece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742117.38/warc/908ddcf1de1130dbbb45cd7d46d5e1a62de21fe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742117.38/warc/ae7d4dc3a29364499a32e9b0b5e3f9cb497e1b10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742253.21/warc/002e40307a370ebe759f6c98bf3222dab69e92db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742253.21/warc/0775922e7ef8586592f4da2ebbdf5031791231af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742253.21/warc/8a85f75e948bfa4a9b15cfcde052c609e2178b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742253.21/warc/bdc6cf7386aeeab52adab313f6b54920a64c55f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742253.21/warc/f321dd0078a26e954e0b366ae79eb1859cd5c4f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742263.28/warc/029a8516cac46638474df51d5c28947ab3be946a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742263.28/warc/19bd7d6c2ae4ffac33f2a0c5bf73e41b1e4644ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742263.28/warc/1eb9d1932f2da12d076c499360771a9396f2d644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742263.28/warc/70bf1f9e6d7ebfdf0dd348ae86418d740505b766.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742263.28/warc/a99d608361cea6263e9cfbe1d8c28dcae58a032e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742316.5/warc/425ca768dbb74dfae57092c96f39d82e22381312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742316.5/warc/5785a3bd8a16849d59698800179ab6b13577058b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742316.5/warc/8f0319fa36ccfa6daec74e902f884083685bfa7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742316.5/warc/b153c88d7ba344e3de7d243a5f6cfd9d18b33b22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742316.5/warc/c8fb80cb2d5bbd8eca252732ff9080bff6712e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742322.51/warc/631e327f4645271c880ff86361c7a4dc85683816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742322.51/warc/9326d343525761618c3e90abead5358d58e81f16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742322.51/warc/afa3e0cec8a89bd39f1227b18e8fb1261a1dd61d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742322.51/warc/eb742333b1f8f943e783a6bac8a121e121923dde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742338.13/warc/5a621f3a54db30ddfd4b94c4398cec21c2602085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742338.13/warc/5e15433070d18982474d9aaf55c7620012604d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742338.13/warc/7dbd46b50f1b27cb36fa3283f983caa23bfbfcfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742338.13/warc/7dc0e3d5a4adcbafa70b3f771190351bd02bc84d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742338.13/warc/c2e359de4bd6d1e649fa2d288760c81c2cae37ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742483.3/warc/154ce9ccfe94225004b80b3e3e2b351b84330c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742483.3/warc/7f0a8f3b53fa004b8e5eb3a0c181e0d1f391dfc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742483.3/warc/aa683faaee389e451171435f77416a3df36c2cf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742483.3/warc/e391cdb820d52ec06e98c191ecc11cc5f6a95f48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742483.3/warc/e3b5dbfcee0df56f6f86c3a8527c569e11c37354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742567.46/warc/06b276eae5c704fc7cd53f02c0e7376638406f5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742567.46/warc/11b5f3cb6e7b0d7b625341cb2f5af2a6855e054f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742567.46/warc/15cf0fcee9cfdb9f3091c0d4d4453e9cbf21b5b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742567.46/warc/229128ad306fa1da7100a9c14c379810c44595bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742567.46/warc/30b5a024d7fc90c41504dbaac884aa40ae9b9822.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742569.45/warc/2ebe419e6d06f5fa74f51cfcd9add5a68a375e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742569.45/warc/59e7efcdf22c69ff4adbd47719a4980553ab63ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742569.45/warc/5f456b64064dfbd245322c2721474d74717f47b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742569.45/warc/bebe98422d0b85924b3011a649044d96eb1a62ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742569.45/warc/de6ed5cbdf4796780d8cf700f2a40443b7c14e54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742666.36/warc/7d9a670198e56fff980c0a3bb7819e3e79a8f3b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742666.36/warc/94e658bc6199d2b0c37fc24dfebf5a3c5ef4672c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742666.36/warc/96d2f5e3e5d2af039c190fa5efe3c9e08a2a5655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742666.36/warc/cc8772530a7c29543e1a4ac35f0cc1964f0bf245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742685.33/warc/067879005e23a778b0dec38fefcbc029cdf3ea21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742685.33/warc/4bf1b84108b754bc72c324e60b0f5fb813c67abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742685.33/warc/66a4a3edbe6ef4d911e2a59d3a1b2c5a85b75e69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742685.33/warc/72111608c02a2f7a1ca4eb857d0832e2bbfd732f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742685.33/warc/ff68b3a24a10bf57bfaeff55ee8f31dd2ddbca2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742779.14/warc/219889b366d00f1982d8a72a1888371e9c8021e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742779.14/warc/5103dbf5ba24268ab82ac391909e1b8ba7b3c6aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742779.14/warc/b6244f57b6d29f5b055210ffdac62f1453164ef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742779.14/warc/bff75fb2d6e8409750e5f4d1b34d56bd272778c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742779.14/warc/dae9bf2dd7768779857188d43cc293d4ad0ca889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742793.19/warc/75bc51847392191ac37a317d8dc6b8b4420b0160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742793.19/warc/77c9b50bd61f3b0867237916ff6c8316ddee8fc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742793.19/warc/8aa6532359c96126853c4f6afc44008afd57685d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742793.19/warc/c821999b42d08ea45ca7d18c09db0462ef132d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742793.19/warc/ec4b1694ad0fa7db637dc05c213048861805c7c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742906.49/warc/2ac6d11a82925213521b98bd34be176ce8133bad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742906.49/warc/2afa6c70dfe312293d8c6c1edd9c084098f969fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742906.49/warc/2f889f341d494468d9e59835e88e00210f51833d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742906.49/warc/836d6485d36318cfb65bbdd70f484a32e31a12c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742906.49/warc/9eb3839213efcdef0b29e78d65f0a83598153a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742937.37/warc/4740dff9d0c63b7b92034322e3488393695d4697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742937.37/warc/49ea6e81e1cec55fd526b1ad11efd0148e779809.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742937.37/warc/d1c935f5eb5d14ceedb8c2c7fddf19f10bb8a039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742937.37/warc/f008e5a5fd5733fb280e4d119b671b22b4ac66b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742963.17/warc/457ea19b1ed86d3d73b93154cc77c3f2d6cd97eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742963.17/warc/6da5dac33670467806ea7f704e3895db7f87578d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742963.17/warc/7a970ddd50689ad10c3274e7bbb6988884854119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742963.17/warc/93a45e4bcfae92dd94d046cb985c7a0dc3808001.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742963.17/warc/ad063d07e13eadb7240fb12509402c1b749ce947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742968.18/warc/0ea2383f9f88ee39a699510eef7b0669c5c55a41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742968.18/warc/4d57ea361d96d42f99b15cfcff370161e58b45f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742968.18/warc/744c62ea67b99554a28ffd282941a6f86ea1f4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742968.18/warc/bcaf1bce9aafb1647323b0c72ac16f357f27fd6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742968.18/warc/da4e599d3ac9ded8e78096abbc52cfbfa10a3167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742970.12/warc/94e301a5288d9e5e8f21ae9a7f0171e5b3664b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742970.12/warc/ae7df2bc0a95d8c5e55b864370bf2e1ff134f548.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742970.12/warc/bb857e937de33e1ce063d5121e15a8092c408d70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742970.12/warc/de6834fa779f4a1ce9b95496fec9c6d46f7ccbc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742970.12/warc/f77c79ec6cb596ab685c4041486d231e0754a310.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742978.60/warc/0514670c4b7d56e4d04804f8aa01f9fec29db64f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742978.60/warc/2191a4cceefe05d2244de03261f627c2b0a051e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742978.60/warc/346f1f8d4ebc362e466a6c7f458452478a912746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742978.60/warc/c633e08986b0fcbd79821a249a4822a57e1dc9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742978.60/warc/dd0de8b2b54508e30d9c84c6f1dfcc1b21a69e12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742981.53/warc/19b3c4b740148a474c8ad65af194bafd7ed22913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742981.53/warc/1b04328b1ed8513d67282ceff958465cbeac1761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742981.53/warc/47eee44468313bed1136728d83c340432b7338ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742981.53/warc/a53c1c4ca0f63e79f98cf4716f5db19e44750b49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743007.0/warc/1eee1821c6bee258ea67520ab1c6243502b10ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743007.0/warc/3bbce5889a58e39396bf1d44fd35b57619f60bd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743007.0/warc/7567d3483db6501976496d2487df181a49e763e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743007.0/warc/97d83722dc5bd44d26530bc62ab6e3cf3ace8f38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743007.0/warc/aa78356f5f17283f685c50b217be479d8a3c9d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743011.30/warc/0a9a31156ddcb2a2d7f627da53ca5606e27d5a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743011.30/warc/23293db5660329dcceaad2aaa5bf1f7231f4dc1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743011.30/warc/3f6510c128c58ce9ee930531db80f843691f4438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743011.30/warc/6eac91a9d7c0fc5cfc8593ac6d3a841351882d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743011.30/warc/d4308fc513f12da83e1a68cb3656d5d166374718.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743046.43/warc/7c6d86c2f882d2082b6fa8844131a569c0fa8db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743046.43/warc/d64e93e1c4b524693252dee3ca6805ae3e4c6f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743046.43/warc/de92b8c629ab37de3690439c985cb4f478d9c096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743046.43/warc/f35aae1fa02ef9a6b10be83d35a7b367b98cf15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743046.43/warc/fa7fdb9a1e21a599979d688d79a65802d8d8310a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743105.25/warc/027562e004ed08a982a3958709760b369eea9a59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743105.25/warc/02d2fb5ee7881a3abbbe9e3b6fc1502d374f010e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743105.25/warc/7907988ffa1c69a8eb044ca633813fc08cb3c17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743105.25/warc/f300d3bf35538ccb6b9147d3f12315118076a935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743105.25/warc/ff491193508d312b0cbc68ddcff0a9d7291247cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743110.55/warc/0910891379f8288e05f7b1002091e546b22577ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743110.55/warc/1e47cfd3875ddce0012721b4aa86d6c322a0a3c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743110.55/warc/3911398a2c813c62518c3f2af741fdf3108f9c6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743110.55/warc/ba3c813c660b39ca5c8d2b2d8af1b49aea7ebb84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743184.39/warc/003a7143197808982eb557e50951fbe8244fc2ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743184.39/warc/4d75afe39fe8d7487ffc5711ac572782e7d939b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743184.39/warc/8c61e286fdd5fd3cb10cf4461cab73178c7e42f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743184.39/warc/b4f779c5520ce08e737cac3fd92abb10eff5fcfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743184.39/warc/fb3be3a6aa7b29332af74ca3b4f75faaae665f31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743216.58/warc/18c79fcb096e3507b2a02c421516036c5e26e992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743216.58/warc/3ac0e3baaebd84cda552a3684aa62a311fd4695b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743216.58/warc/5358408b6c5406d4e5b8ce25ec9e91c45afb456a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743216.58/warc/ab5da750e574792269da02bed941efdf4d528d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743216.58/warc/bde743a33ea683f9ab136322f2fe8b8fae092dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743247.22/warc/1249b704a2a32cc6da6d45521024ee06dbb49ce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743247.22/warc/5dbea483179d2c05cf51f7a4330a0ce6b1c7e0c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743247.22/warc/863904c7111cc98b276c0a8ebaf9ed139883a90b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743247.22/warc/bdd38484f71e70ec1e940353ea073c2ee9d70023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743247.22/warc/eaaf70e8bef04709ffd705c8564800605eff81ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743248.7/warc/724ec54ea2e47d8eb0d5f16fd885fff1b3524589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743248.7/warc/ac3222766d9b4d622f9e9542738c029c4c7790fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743248.7/warc/b6b26965576f8da6be0c57981d404c797ad1803e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743248.7/warc/eddf314a459782cc9478b2c55351b8dc2d03bef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743248.7/warc/f7b637c98b088e82c65254388eb8f19c5a18d484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743282.69/warc/4d6e05f70c2eff82ece7be26a7b10350d5eb450b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743282.69/warc/6cdde59e89874181184882f0a866148cf7bd6767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743282.69/warc/81be99f4a22c6adbf8d2f86cfae26db80bfcd78b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743282.69/warc/c5d013e54122ec2fd8968f025693941e6e251bf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743294.62/warc/4526be01cb1590c5049fd33acbba1ef48a08c24e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743294.62/warc/b5c9c9f80209dc401de4efcd99d5b1262afea1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743294.62/warc/b81e05c99c7db8413daa0a65f47c4bb3e19ffbc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743294.62/warc/b93133f783cc0b3cc4f484a12cbd2218d6fbe16d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743294.62/warc/d5fe26bb2a7e9b8ef145a52fbf92bea396deb6d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743351.61/warc/1605eb814aba8ab8dba377fa416bf7976c16936b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743351.61/warc/6082ea5227ca38ddbecaef34012ec07fc76ffc51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743351.61/warc/84894700f7b164b230be7c049dc9fbb64f330ba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743351.61/warc/8d0ad1b92ceefede2a9f33951d9bfa873cb1e6dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743351.61/warc/f01e51216289a041d120f0b003ee2ef082aa6cec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743353.54/warc/3e3ab57d58d141ae7c25a30f34772628e6c26ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743353.54/warc/6f1247fe64b082ed22916936edc4374eff347b8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743353.54/warc/7c61d36b3600ddeb8a74ad06ddb5d09f3941f7c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743353.54/warc/8a8ac1ddeffb939eaa40bce0368dffad843aa278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743353.54/warc/8f46fd434329c2fa04f4ef5eeba43469e71abd51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743521.59/warc/340ee6a5e630483f318c5f935767f783c604f8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743521.59/warc/5740ca5792f44acc1fcf9fa70f039640238c4dc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743521.59/warc/6f994b1526c44583235285c36d86aa79d0eeced8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743521.59/warc/e773da18d5efcc77a4c0eda50781066b13fc6bc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743521.59/warc/e7f4e0b0070bd08328cc38c6707ac6667e3cf4bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743714.57/warc/35c71c589ffee4a41f0956b44e8b0028b42046bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743714.57/warc/37b95cc81af4b76bd7c78985b55e4820d3c8692d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743714.57/warc/78170c32056be471e8040f7833d50ddd7f1d4eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743714.57/warc/c2b1488cc221a1506602dedeb21c983fa1a112d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743717.31/warc/23829ffd66fba2038465781239ae3389bf49ae40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743717.31/warc/32ad31171297fb0d92af44e6943a2a38ad085757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743717.31/warc/52ea2f5246f5cde43884eb2aaa5eb7a4632bd5d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743717.31/warc/de4b49f268862135fd9d737baa253f7900c170a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743717.31/warc/fe7e4741f736244106f18ed448e594e146a1609c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743732.41/warc/3219d0a1e671c5952507ee111664de19f81ee918.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743732.41/warc/71055784007a41f38e0517f551090a4d9d55edf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743732.41/warc/805ed79d89a3884f2eee7776507e458aa6bd0279.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743732.41/warc/aa98d7430e796805d49b035afcbfa82b132d8065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743732.41/warc/dedf225d64663c1fa0edf405bdad937f79415452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743854.48/warc/16c5e6d03df586e225ed9460b13a5c4d6242fb5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743854.48/warc/80720a3da70e92552f149c34b19388b91fc2383f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743854.48/warc/8b404ecbdd567768ee7964a59441ae1a349852fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743854.48/warc/aaa1074d6397e68d0a565d8f6325c837e8cb9a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743854.48/warc/f26bfc0b3e64635e65acefb9943bfdd1af1f9b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743913.6/warc/135e03eeaccf2fb8cca054721fb3cfa8d3af24a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743913.6/warc/2da50fac2a1ddd78b16faa4499a8e841845539be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743913.6/warc/3ec6df17631b8345d2946f7074d4399ca861b8e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743913.6/warc/ba093320817c6d3ab7b842418ff43d22e55b2425.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743913.6/warc/c4cf9f8b8495a1cfb9e2bd85fb8dbaba9837f83c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743960.44/warc/2be5c67278d0c8f4b3d8c4842bce3973601b8af1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743960.44/warc/37386560590a24c974469e22b304f2428aff7fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743960.44/warc/9b39f7ef037ba62add809054ca6ffea823bad209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743960.44/warc/bfc67857898a22a2c2f41c8c62ae64b164765593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743963.32/warc/93f6e77adccb690ce3dd6b448b37aa4f96d5f72b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743963.32/warc/9ab2fb11c663d0a0d400a1119a681f7fcb4356d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743963.32/warc/c4e99bdb623184613657ced37db413cfaf0d3aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743963.32/warc/ed399bad9e8501b6c211679e45ae952136acc102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743963.32/warc/f1ecc62d811e17a360ecc49abbe429790e6afcc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743968.63/warc/2052f8e3b0d3359d1f257623de9412a5f0e273c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743968.63/warc/431855f609ce5b1759eb4bdd8979cebe7aaaf8aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743968.63/warc/6eea3d2bd7561cdb86a66e67ee1010ca83e6838f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743968.63/warc/a8f04e2c684d5e29e5f53b9d4a0ebd7823816590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743968.63/warc/af7f6a3f8555404c744bb9b40d9d521884b14e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744320.70/warc/098149a437346bbef171afd88923f8052c90c936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744320.70/warc/2236986096e2aa3628b23c2480c51fa0c67dbeef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744320.70/warc/5278ecca39b4ef368605bd0a18923e5660b4a26d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744320.70/warc/a2659aeecb7b1dc3b7c32ddf07a7733f811cae8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744320.70/warc/f3f0691d6e0b7b4480fc2bc1f60ae382e3fb4d23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744348.50/warc/2cf689fece804bfd41da8e66c7d4a71227fa8b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744348.50/warc/5c2f274440108508afacd9d70fdc0b94573b7337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744348.50/warc/7ecf703503966b2a1fb73a5d3161f2bf4deab846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744348.50/warc/9092d2506dd7c3a0147be606b544ca457ca412dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744348.50/warc/e0a9da4acd93e4a7b5c84a8375f0f0ab7bd70770.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744368.74/warc/4498476c297645e2672d6df5f34db215cc02c62f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744368.74/warc/455aa00521245e54b984fb14830b03c982f9dc57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744368.74/warc/5f73a49541a2880fa65595c02413fff432041a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744368.74/warc/e5eb73206028a5752035b8b77044e9fae40e4d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744381.73/warc/780d0fcfd96bdb3627b19a732ed46b94e43fefaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744381.73/warc/af45c6bf859c8f16aea9bf1129980648dd473d11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744381.73/warc/afede36df4c81cb27c0905e1945a5fe788c662b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744381.73/warc/b3d17f202405d3fad21b313682b7cd5986bc1d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744381.73/warc/d618317dc9074443d896b3f52e48570fd1ee2bda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744513.64/warc/16746278557a6be95c5f5b84d975733f7dde06dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744513.64/warc/3ce120fde4734a07f0f4ea9e0f97aa6e1c9b8eae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744513.64/warc/59956cb20cf75a68f9f43808173bef22d13f936c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744513.64/warc/8cc242d207ba4ff15e2cb51b8a0b8adad8ef1277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744513.64/warc/d997a23d7023237761172a8ffcd5d8bb05bc73a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744561.78/warc/00a103446b8980f06ab3b4e87af5c63143d7816c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744561.78/warc/0701140b2331be4c6cb96954a192eb10617aaef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744561.78/warc/9f3263e40af4ecb2005045ec1e3d05afe141fdd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744561.78/warc/b067a47ad4e1eaed15ab36b172ad60f5d0b28253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744561.78/warc/f9951d2f66fc3cd96436f190d8fe9cc2df2266ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744649.77/warc/78b5ef43597d4d4c2c6f79e4c8cdd26da779dc7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744649.77/warc/ac495cce47f3cc6173a18fcc83007a7946fc3a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744649.77/warc/badc6bfde173b8ef0352946d6b828d0dfda2f9b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744649.77/warc/d19759ee00216c4986a9b1de0d0a58054b98f463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744649.77/warc/d38a82e7526ccc0b9d93a41a2924695c8b544b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744750.80/warc/1368c00fa77f46720c787b4904942871d3e33d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744750.80/warc/616551d25b5486dc428f25fd30fef197968aed60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744750.80/warc/6811b086b505fc8f0404ea0ba1f8ae6ce4c7b8c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744750.80/warc/916f1dad031a3e647df2de25f7970c166e9fba94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744803.68/warc/0d72db386f027bd4d4f6c78346f348b68c52be93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744803.68/warc/68b507d875d938724bd39bf89e33b303ef38bb10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744803.68/warc/a3105d9c048fce90a6fa9323f2910bfef863c9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744803.68/warc/b845a3935a28051252015197bac359520de2715c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744803.68/warc/c57d60e974cf423968e95222ff512c815b984df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745015.71/warc/09de253e2686039c9a17bb8712a6f002b5f9a001.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745015.71/warc/0b9a2d57386de445565e369e501667a8d5db86b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745015.71/warc/6f3daf18bab92580d596606e82cf6c1bd242bc31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745015.71/warc/a02bbbf5f67b7935b35feb8838fd5bc5802abbaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745015.71/warc/dbc836b4444298feb1513bde6afd9fc1bd802d89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745281.79/warc/3c451fb6d1d0db8c37079c709905613c6ec597ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745281.79/warc/473dbf1175984ee09582a580a0d199cd90769bdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745281.79/warc/96c8b37e7422e08c3f2bdd61a3c88bdad0c10497.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745281.79/warc/b971578118ebbbaf961037a0b4a4339fb701b257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745281.79/warc/f169301512bfc04bb5ddef11269eaae2a37aec67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745486.91/warc/35967e427216f9e38c421ed76eb939c829b41d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745486.91/warc/833fc713ff91c7cdb7a6a801f63f37da1ae8fc70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745486.91/warc/90359c186238d8c8be6772035ee5334cc9dc83a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745486.91/warc/a64533388dee5819a81b5066405bc2971f82eb75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745486.91/warc/e1b398aa9b4561d02117cc51c552c7d6daec6108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745522.86/warc/2eb8906f99fe367c1adc0bae0d9b6d29d74a0661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745522.86/warc/8bbdb301ec6b3eab73ef0e516a5c7a0b519a1f16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745522.86/warc/aaa20499a257afd960862d9a240b232580075867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745522.86/warc/af317884b7178247413f24c9d247e00e7e3b4d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745761.75/warc/2e732d3ffdba2bc797291fda6120e6979d64c3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745761.75/warc/8e14453e932a6093f9779e07b422048da65a02c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745761.75/warc/d121370f76a473a5dcfc4fb9875d1033cce5a1ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745761.75/warc/e9700ea0ee2788bd8166849bfc7cbd632a7440e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745761.75/warc/f0b8772206e445e3eed639a984665230583748e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745762.76/warc/1525b3f558c3c85e9fe85e9d3e1e332545590ce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745762.76/warc/1e443f09048abb30ddefadbe59cfe8d42b9d24e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745762.76/warc/3da5f69bba29edddae0244a522409879684e90d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745762.76/warc/8dace3e941bb0b7259132fe4a4456e5c470b548e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745762.76/warc/8e2aafe9b3ec8f38d3f44ca7a656455db608fb4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745800.94/warc/3e6c6c567ab8cb8935a71f55b3abd329d7b6b687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745800.94/warc/70753a696da60ee77ab93d2a593c5ac752b442ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745800.94/warc/846834cf3362b8801acb1f71b9608b8393418def.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745800.94/warc/96d95ab78463a28ab4371ad6a357cb2068d8959a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745800.94/warc/abe5104218e801789c5bb8377831d78eda256b28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746061.83/warc/44f26d5b530d9818437cf9544a33b0d41a3fa519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746061.83/warc/4d72711f1fed93dc381aaab5c052883d56c72450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746061.83/warc/806e6edd9d59343c2c6d3415dee6281c050ea468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746061.83/warc/bf6c10fd83186ad7176bb46bf71cd9d412254b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746061.83/warc/fafe6d5bb4680db58504a4437929d7b93eace764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746110.52/warc/29b533fcee86073b0a8028c2f8d2d5cc51906d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746110.52/warc/43d3b1db43557b4a02e2030f23a837a40d1f09a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746110.52/warc/938d5b8624091be060e7685ad895c28e34014bde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746110.52/warc/9f9e20c5bb87ff40d854498c581bf129c76b55ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746112.65/warc/252e58dc5193e1ea5ab77261a742edb440f8ebe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746112.65/warc/a3574e2949ee3256aacda93cfef3187a4f30c9bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746112.65/warc/b720d616b46ea35a16f1403a5540876c21126a2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746112.65/warc/b75223baea7f48f0d752f42f5421843b55ecb753.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746112.65/warc/d63c5e90b1ba6016c1487ba7842f008f3b52c8a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746171.27/warc/30b12e352cb9fd67a15964b3dcaafc6c4a41410a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746171.27/warc/57110200f7f4d0c59aecac3f8fe2bd361bb401f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746171.27/warc/8b0e869ccbcc74a1083953c5fa42f48adb08cf80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746171.27/warc/9db9fc4b3c9e3747b56a14cdba64a79202db83da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746171.27/warc/ae22a491aea864999ba2058dccb28d7fe93d3af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746205.96/warc/117e3e839cb6eb8e3ace400ec9b568a9b8cfe1ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746205.96/warc/58c34d4e08a0557494fef319e616573a7dad5586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746205.96/warc/a4144a1602b7114c651eb413328f0f4605c4f075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746205.96/warc/ce6c535c615f1a441c99422a23d8a0af7ea66fe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746205.96/warc/de367ed52ae724a1d7f1887685b99f6b41ac5d3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746227.72/warc/0e372d80ac1997ff486e5374695760eecee564ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746227.72/warc/8dbd4003b8118e4bbf208dec4cedf7a5d5d9c379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746227.72/warc/92ca41527ca4f6ef8ed0d44e1dff3e35af490013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746227.72/warc/a66b161dc88544d259c5cf27854036a38b129c75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746227.72/warc/e93ab9270364431f0c38b64282f5030cc080d19a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746301.92/warc/15113460e9f8bc3363828155c3cd9e513b6bf651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746301.92/warc/3436064bb3f0c1ab96f250b6772d31871e92b813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746301.92/warc/ff19c88e90afc3259493dfb3089a22e16513f744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746301.92/warc/ffae6a41a592b44aeb9db191c667dda7f7c6ea12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746386.1/warc/3a4b34f859750460f5d6d1ac864253941273a59a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746386.1/warc/3e1632e8bad00e841b0ca8535e4e2ffb2299ba46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746386.1/warc/5ec91ddc2ead72625181a181fa959faef5138f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746386.1/warc/d1461825757562b62eff42a07b80959970c63cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746386.1/warc/d14faab25fcf25c5d90408c523d8522392245646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746398.20/warc/4964171dac7bcb1a00505aac98dd4a53450f2d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746398.20/warc/5919fe6bca69728fb511d41262c054589a58d4db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746398.20/warc/5a009dabc1cbc753ec476743b1a761072e44d5ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746398.20/warc/b74b72396add0401b566d8318197cb9f6caac165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746398.20/warc/b88441fc62bf0be0223f80519622918379d21e5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746465.88/warc/1373f9cbee6bdf7f27e986e205c0cd67335a5b52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746465.88/warc/9c05540746708086d2666047a3f73966d8e4429c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746465.88/warc/a32b4905e6c92361a7094b8ced0615a7156d98dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746465.88/warc/af088fb8c5e4ecc00e4b81c87eeef33dfce9ab88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746465.88/warc/b11acbdac0c0cc07ee9ba1b820e2b810d086b619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746528.84/warc/29ad356eb4aba107f0664d3d68aec0fb502fc22a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746528.84/warc/4e435d4d413c14e31113057752463aae1e47887f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746528.84/warc/6f241efc30a6a69ff25152ab30bfab9fe7b64866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746528.84/warc/efd8e5f3497186c167de6e8df9ca09f7013087d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746528.84/warc/effeb5bf2c4225478bc068a123446f5847e48207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746639.67/warc/2f006de7f8d6c15fadaa67a675d6d502bf43b094.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746639.67/warc/da4079bd7cb0c790bf2ffaafcbc6abbcb8c9b0b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746639.67/warc/ead55650bbaddf94902ebe1c7944adf17172e167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746639.67/warc/ed331a874abef077c760a3b5fe7ded19c6c65e52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746800.89/warc/40dfc4e38a09f62c6ec16c92bd36817230e3e0bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746800.89/warc/4fb692b8de5d70423a410c0f3bf6fe4f7480f69f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746800.89/warc/658902d8095e4989484f2d5b6adc83f1a31ba7d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746800.89/warc/95f2feb1c71e804a58578d37fefd4b6740575c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746800.89/warc/f39bfd66b10291d8adf68d72a14ff81346d1dfac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746847.97/warc/2dd2f57344be830da76bbfe91f0ea6446b97e3fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746847.97/warc/53180238d69690f96889a14beefb2472532a09c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746847.97/warc/70c6e1c7606db2420f6169f57cf7757492a85c13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746847.97/warc/c8fa0a1cf45d3b66e037c01bcb172f4b84cb87fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746847.97/warc/f2fc122674a38c7e0bb7e69b4ca78223c0d1a798.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746926.93/warc/005ec3ea6388c09175d3cc2c50a1bc219eb64e43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746926.93/warc/176b5948b18f29e557d73c390c7659ceb4997c6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746926.93/warc/3eb7dff8743e5f3d304a5a001fac34e920ce37db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746926.93/warc/950cd7372687ffa25fb73036c6e0b85af8088456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746926.93/warc/fa84b9610ea8cf49813fbcf2f2d9bc62c2ac9327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747024.85/warc/251fb21e3c504739d239b11282a1a0a6df62bc92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747024.85/warc/3717ee0593c6286957607269ac7a057976556460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747024.85/warc/4505882759b42a943a1c536e9e0db49c9d547ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747024.85/warc/b4b7639768b474281a42e2d8207240756c47c8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747024.85/warc/d97aebf314ccf83463b8251a61e565f1a4ba65b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747215.81/warc/342e7bc2198158fd269ffb4cc75420e07274568a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747215.81/warc/5ae7668704ebd8b04e0a0e121c631b2cb5cfc8a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747215.81/warc/6dc8e855c3d6578af0f0f68c2d25c754196dcb94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747215.81/warc/74d87836c62083a978861f1f57f58bba6f4de37f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747369.90/warc/0b767276804d73c96ec7961ac44f0622c7ecd46c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747369.90/warc/64807a969c749fbb215bce5bd57e0eb68b4f870f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747369.90/warc/9e1d5d3a471f3969f9208d2991788bacbc5783e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747369.90/warc/b0414a2f69389f46e58f5201d1efe7808e876202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747369.90/warc/f3f7c1d1170ebd9d500322b730eb7060517d6a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747665.82/warc/064b4586125b7d34ced8cb16fb787677b54479df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747665.82/warc/289c098d68b8fd3507592ec87566ee7caa823ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747665.82/warc/4f60552e0527f6c6c5673222c40be9c8d02383d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747665.82/warc/64ba2129ba14a7276b16738458bb12a88c481a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747665.82/warc/a0ee9d2fbbce0043c79e4db424fa8ccbce13bffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748315.98/warc/2ac1ba8c27edef9de2ed6b6eeb571d5eebc9e105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748315.98/warc/99a6647c70a4092d9042310932cea9e353a9a012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748315.98/warc/bfa22a56a29ceff7c87989ab8f773d02a8b9defc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748315.98/warc/eed771d274ed18b5ab9555f09e9b0ad1021cb172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748315.98/warc/f182c4c85c6e38086170889764636bc4ebe33ea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748901.87/warc/0dfd10eb460bbccd40e1df45af96b1a909f9e419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748901.87/warc/1fb70dbbe8e5d6e63abca5445eb5fe24984e6c00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748901.87/warc/8609d68f7492f9817149a56b9274fbe825014d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748901.87/warc/d83d3bcc6381e5c7344f2540a1fade594e1f6dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748901.87/warc/df4562f716a6c927f93dbd976f1f74a5a777dd1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749054.66/warc/5754ded070a23f40090b6cb3b774a1476ab99cd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749054.66/warc/6414f325c4371ef2954ce553fec8295a6939d4f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749054.66/warc/e8852f0f135b762aefdb424341a9d878c9877350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749054.66/warc/f852ed8ed7e38ff9074b3516e91523ca2c919829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749562.99/warc/075d4b1f0736864bb906447cfbe68066f408bafe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749562.99/warc/23f4997a7e7d6115a33f5c223d5d3ecdd853107c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749562.99/warc/99b6d6f78620c7744a37c8edbcaf26c2918db5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749562.99/warc/a4ebd8301430a7dd21b2b49f142d08f1ba451496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749562.99/warc/fedf260c34b8d31c66488a88f564ca4165f0c828.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039750800.95/warc/4729f9af2278e5c6a5d6b13f03bf8e4af8aba41c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039750800.95/warc/9a55457855da3baaf79cf806bcd892f30341e515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039750800.95/warc/9bf9f348f8309c360168fb225e1e8424dfcbf0f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039750800.95/warc/ca9f2427ae5a9455ea6d68eac7746f1df2f452c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039750800.95/warc/e073e9e74601743e9dbc0a210e6eb063a96ee11d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1742b0b80b91b01c49348b9cecdbbc53d8735b12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1815e8bb03cce18d2bd31c6694444b7e98257444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/190a92b692f2811a2085cb8490dc4388044598f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1a6906102182cee3c833e28617b154004a4d5d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1e01e23b2f58ccf4c18063e8b065afbff336818e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/2205166a35a0d7368cecbb391861b2df9999107c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/226ef60fa42769e8b1123479eca747e19e5e12ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/267a595448dd93d5d97938181c9fa9f86d73c1e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/294d78ad44a9bdacaf3a995429af3e84d89c6e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/299070ad45207016043e92755d32095e6a691f82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/29ee7fd360183b2fb72d53e317465d053cb8598e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/2bee311c02bafdec01cb4fcd2c9388cf58efce11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/2bf3ff3060ef3bf8f7da18d51831f948a1482a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/2cd029d9d9664fca98afe847883bd72db3702285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/2f9f1a108e5754d12544087ebbb8abf873a96bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/34ef348637d01738e946a8f95c546ef71c66dbc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/357f4148dda3787aad3a3ca770e8d6431b7dc0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/3fdcdcba3425f2b8327d6733dc09ef3a5dc2a998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/44b26c64c6b080bb97ed6ad934eb466b92f7926f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/538570ecf76f6d2a465bbb7511e7e14da0f2aae6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/53a1e1b7cc89f484852034e528843693284a7ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/54f6029b790b454fcda75f0693af83427f59ca9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/5e8f995ab435e95ecf196b4e3339cba19ae94dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/5feccc32f96d5963ca72641d16d7a7054021e05c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/601911ddc16648e947f61572e8d81e886d5f8620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/60f1f7e6a8a551fb0707036128d257d9a3985156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/6751f06b7f49f0c91feb2e163c53122761dac3a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/67c69a765abf7407f44254c68000c9db8c6ecea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/68ed6e1570bdfb12824a84d7eaee4282763f2875.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/6add3c0f60899314723843a181cf156b3bfe11e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/746c34ec20b3846ec8814b359dc52a9543b162c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/77b1777eccc8de701a73bd628762e357b412aa2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/7b655e22647099aa61e10c773c8bf6e9b8c4588e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/7c067129c034f1f388e1bfc0bcef48a76d762160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/81be23971b7f72b4282fe84379aaf35938c82c23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/89503d3521db30d8ddae530779e3f3cdad28ae7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/90b7f57baf182b9d8c1f6df0568a5bfbc1023d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/970d2070a6d0b8d5e08212f039a16c0c1399fc7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/98719e8c700efccb7e81444c2dad6382610208ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/9e12518b2ae191ab6c040b9df2f221a0a7f867bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/a520f4203b8aa60abf057a817cb95a600cc60bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/a99056fc233e81a0b841720364ed9ba8ba774b11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ac3be0ae35d4863d2f74c16574d556013e3c315e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ac9f817053ebea58cdef60ad597f13232dc7cd8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/b1a69f49f8953db217430c7dfceefe28807f4fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ba41ee819a8e65cb41d9d44d1303103bd6468ec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/bb1d2ceecd1e7dbe5e6b817c79d36e1b593ac651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/bb1fc9d2d2217f1b63ff86bebff5e2fa45b3abc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/bd8e22840a90fbc2c57c827951e9a9d79a66b44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/bf19bd3c0e0c9e6c9b5a948b58901358f2c54153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/c881c0fc3a6eb270bbacedb31efad3053a194e01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ce0b06b42c285887aa84b40abd67d18a723a87be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/d101711d38633b6afaefd787a6b5653925f66643.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/d23dd0f9061715163d1460d7dcac00417fe16e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/d395feb767e0af02a25950ae20c67ca31bd816d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/d49d4226bfef97f4cb19ce77a0a4c0f7eeb0af34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/d87c26c5479f9669e00479b8d7e9e5d90306a224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/da05102e63bfd76857610a1958f520febafe9dc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/da5f43564899c273d8a104685b25c4f3a174916a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/dae7ce9091cac2954e2adc1e128f860a61369dd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/db97e8a83a8c279e6f597b2ed6b1082fa1559436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/de13a4eb7bc9c5a4c4df532e2c02e9fe7aaef930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e06760f14d9e42dc996bee4e99f346d403dee3c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e19d6fe8ca3121f0c180211879578d06ec918454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e3a654bc69ab788192f5e1bd85c24f54b10a1376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e48465fe93ed6a0174b6874666693800c77cd904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e4ed248274a2019e561547a713ff167f48c2342d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e7d49c678c8aefa4a18dc2167393838a5af3b62a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ee065341c310b498550fe346bbda0501fb5ce9c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ee7b45071059707a7b32cfaf4dc03d740c9c5563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/f0bc5f4831792d6a38b31631ee8f95ff92097fa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/f0e37c9681951361a8e465e8d811be7def92b474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ff548723fe4a3004d19a35ddc6316494cb58498e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ffa8446f2bbf2e2638b94323e8fc50d41da4698a.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2018-47/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2018-47/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2018-47/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2018-51.yaml b/configs/cccc/dedupe-url/CC-MAIN-2018-51.yaml new file mode 100644 index 00000000..30515f74 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2018-51.yaml @@ -0,0 +1,660 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/00517976eb757f690b33f0707cfae4af288fe928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/0111a1289e97f1454997d399e0451e0ef3c886c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/061e06233b94c203c1ae74fae20dde15eab42b12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/082a51f8cb0f552f4564e8d8239d0a7ac602d7f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/08e07b70432419266de8e23f716f6f58c4b4a850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/0a6836d1c0b38d98a53138eb12c2bde22f8c25b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/0af57145d4f4651b8d040157a977118c278154c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/0d13aba5d586b9b77c52c9cf07d75ad9875e7fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/0e5cc3fb0a28558846acae2ca46143d3e8d04cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/11cb201601a761fb0a3953a0423ff8d681cf9027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823009.19/warc/5455e79fc12db4abd935a9e21a56a45dcd88322f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823009.19/warc/61b4c0e8f2080429a739effac29a0f9bf9d6008e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823009.19/warc/6e7b76693ce0f9b105a1c251e31d89436ec86053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823009.19/warc/a46249552df19e07ed284985e566b6cd2f7c867b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823009.19/warc/f00f4f65532c6a3392814c7a131ebfd087158174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823183.3/warc/46a8081216fdd481e5e089830b4e0b202ce396d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823183.3/warc/5b776b9f7747fb382090a9dff106da2af187a059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823183.3/warc/8bd988afae1f4ba68858b810b389dd354cb7ee4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823183.3/warc/eaafa11b4ffcf0046969bcd8bfc6a060447f590f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823183.3/warc/f2a2a52da5946ed99f4500426287228d7f3d3ccf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823228.36/warc/56b4a27012903ce2db69ea547a95de1473ebce4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823228.36/warc/81abf17657c71978ed0f5046d09853ab6ab92e7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823228.36/warc/9660d0638fe0a7a924e4cf0e3d8decfe18986710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823228.36/warc/a77d1f315ca12d523b3fa1081ea4cf81a58ffdd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823228.36/warc/d411107f5fca72fd4f5b13ff8a10fe6417a3b96b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/5e50cd9f9911ed185d5863161c272f5cf667d8c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/655a5def0714112f9399106764f5a6edb8280b00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/72cdfa60db9b9309a6e96f99c984f9008e35911c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/765a3096a92c9392a05679d316e0ce8aab652e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/a4f4748449fc850b40d00d596a42cbdfb797f65c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/e9762180d5bed5313e296e1fd29e24a719b075c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/067231addfe5236f5ff3e8b1a6cba9870a3b461b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/6837a611efb704e04b90de6d420a5ab52ddbe23c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/736c48aa44703e0fde68d61f97746708ee070fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/dccc8b019b0f045dc44adb62b01b5286ad811301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/fa58458972467e22e6acf9700ed8a244d7357272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/fc497bc9f30ca826eb177af82dfad1957be50fff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823318.33/warc/203ad4b7a880108fae59401a0ab8010535138e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823318.33/warc/2824f49f59966ae8d2cac16ef9d581c704b5098b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823318.33/warc/48a98e3e44dff3fb15e3097b6e9b314a5ecfe0a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823318.33/warc/58f50de1c736fbc1acb6529c61319576c56c2e21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823318.33/warc/cda3ac5c772389ad117bcb34806539f046dfcbc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/20d5b5d8c8a62a72925cf322183c0703b9072f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/27beff01db814559b92864044357bbe233178a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/9502e1da75a20582690d9bfb9cf017470dcc4980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/a76c6fc5249e6f21c1e84f97ab4437021666e883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/ac62258a25152563623dc420df99e0ee4e720e24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/be984922e421a9902068c646add247c771cbeec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823322.49/warc/06b0955ee4093a1baa135ab351e7335388c56cea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823322.49/warc/3461668deda0f712b4cb5a231eba018f28be84dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823322.49/warc/4db87483029baf9e7efa7615db828344d2ced002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823322.49/warc/7d053501833980352d9da56ae35ef3b4582e7fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823322.49/warc/ec27b2ee7b877dc8c1b7561a591915ffaa0b1304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/529c1e19078b22eb877ab282355f96fc9c51f1ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/68dda93b01c8b464c3b6692eccdb31702d650371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/995b63812e6390507b133e054f5306b51a798941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/a0b2bcb9568bf827007221018e94321207cb89b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/b47747cd5106f453d84c7dc9a78db938954b4d29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/e431975b7abe6ca9da99f3b1971b0618f7f36e89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/37240650840fefed818d511a92048a5a4908774f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/47700158cbc80fa135adc200d34a1860e5c9405d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/5516042da1a52ff5e56146c45506901bdf9f8896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/585b6aabe00de6f18a31274ecace2b383f1d6bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/a571848a8494342a65567e8b40b4f0307df7dd5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/f816785f1aa4054152b02f38d76e34d473e7e139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823382.1/warc/053c61a61d886e06c756d6eb616f32b30907fa3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823382.1/warc/6c1a86a928212360e411ba6c17313d42498b0e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823382.1/warc/cf7d2df6f247ec415870ca59eca0533566187f53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823382.1/warc/d12490954846cc200f45406a441e1191485040de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823382.1/warc/db2c3449546c4b8e3c029c7412cfd5061ccc16cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/36efaafcdaf9e4d0de3894ffbd2305ab9a68ada9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/6cd9f20daea10572e33073dd57039fd8e32ca4cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/a50cd08a0d86ddc8d44346e02ad57aa45e81417c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/b1fb9a568a19ab12e16c61033110551947788543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/c3f224eed075aaa4a911ad8cca73721dcc8ae6f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/d574ea62dd6532647793b2863656e9b5a17bd6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823445.39/warc/0a4f3f0649cfb0de9c0cf8b7a7a5845df1d24b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823445.39/warc/1ef9fd511489257c7039bab70d5a5f6e1956fda7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823445.39/warc/385cf6c046840b0d2f24df8e7756acccf51ca191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823445.39/warc/79d136b74a7510591a55a03edd01f09fa188fa35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823445.39/warc/e5424e3fa956164f4b6f0202d9368062218ce00c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/4f7d3420194c3ac19f4fb7990e0b77f819fee7b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/62f4a7d4af309c6a17c86defaf13344397ad148f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/6d97a5e98a736e4b90514f6fc5e6dd1231039dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/924fe3ae5c64b2f22dcafe816fb90c5e3935a036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/b149386d7e5308e89ece0312cd03da8417c572e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/ff4726d05048daa81c1afe4cc019d1bd0f4d3113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/0b6f45294f3976dfe008209828d6508dd27c04d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/13eacef3677744969a78327c8612a82d9fa6c0a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/7062d7964354e16a5ccdabb7c5bef9df2c3126cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/bd2b87a07c3e7e9da232cd564ffce6a80ca7bd02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/c4d1e213e75480ee831f9bdfe6116a5f4fb3ad68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/e3c3396d62324e1c1eb51674b2cae72018c061bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823565.27/warc/05daa45182bb18b8e3eb8a0ea65c594a9f5c4484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823565.27/warc/4f665108a76b57a9cb1c246c44c79df07dc40b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823565.27/warc/52f8c97b33a6eacb22d18d913991493a8ad309bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823565.27/warc/76a9444ae88f778b68093c9888f2590fdadd4a08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823565.27/warc/e0b34e6d9555d043995b475666d088d5a2cbdcd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/67a3bbd1a38fb69389cb9b7d142f37c341f5fec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/75d04fa46817f869df61a7530ec4000a88107538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/d3f64c8ce1f4a48b3b0acfb6a6e8cc0283d2622c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/d5deea4a5997de14d119a05b034937b5ea97b8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/f29b4aa5e334a5c3cd07c2c3a57cb0607334b92a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/fa6d4369a356c1b0c4014540d205eda083fe313f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823614.22/warc/0bc5f98832b6feef911f4cd251bfdce3b07329de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823614.22/warc/28ed3fd64e90a9d13cbe1d80057a189e7d18da37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823614.22/warc/8673190c4cdc33ba0b9140d15829232eb9d98c24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823614.22/warc/ca4452888fe5dd3e7eda7f4343f2488898126360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823614.22/warc/f9a5a1ace7b09f8b33d28f04113a288c4c1d0f17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/23ba24bf83aef58d22f13bd04807f2b6e7ab7a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/399d28c86dee9e5eee65b6d025fcb699c2c38dfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/6394faf8f8d8c15f484acbabda0f107ce25e1745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/b2f0a52e75cb298cea3cffc3ca197c040123f8b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/e0763748d49651e95be77368cd6b33689b9b520c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/ff93bbdf095feb022bd3d92980a719f51dbbd04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/171e8df42e98e0ff80b0629bfa3a171445ee23a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/199c843da32d2a6b2ea44186e620785a798dd916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/43bee05d2638bab8e439f8216004ae695eb6f21c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/be569afe7a5de79088b0ae4b11101866814fedde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/ddca38627b472056eb7ac65d6b953a4a18298170.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/fcec3c1f81e901484a1ee879b233fa6ec13a4580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823657.20/warc/0faff119e74b598cfaad83593c1871385202d417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823657.20/warc/50759bba2603b522aaaa44cf318fd7747489308e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823657.20/warc/6419709d1293609aa238655776d8949bb4555306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823657.20/warc/a94d486b50a1149cfd43d0c0c7890ab46b083e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823657.20/warc/d29d78917899976da8528d7942580b1ff44f6b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/0f1f53d22890d8f33532a332ab3982f3c2f512a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/2dc7b551673fda54594682ce68f9bbd99fec631e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/62a696378d54f7b97e81c3f0ba3ceb6c826fb9ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/64b34723ef042954a0705ed14375aa35659c47ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/a3aed517d88b6ddd4708d13e9f6f2acadc013fac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/d543ddfddaf18337482b3bd531570ef7f61a8921.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823702.46/warc/481c7d10127abed8dbcea2a567d426ba5b14a87a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823702.46/warc/b2f4434095b2f880746c63edccafc4f780f65a9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823702.46/warc/bde95529837b34cb959db6018ad1ab36893e20cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823702.46/warc/c18fe1fb49079d190c65b47422bb99c90c695bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823702.46/warc/ee287b41b618b2ab0feb15c5706b2a0aad79eec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/72216f9460a61a7cdd0cdab69368a15ce2fd2a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/7c311276e62253bf3293db45e4d23fa5d3fbf99b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/84c248703922ac16069f8b37c56c31dfc74eefc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/acd8b3684901bed8acef613b1fdcf09c37a7add7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/f4817902e20a6710abe174e03fc9abe47e79a96c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/f85b0f0f893c09b485e86e965e906a45b8c39b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/128b937bf4d502f0c366de7ce319dacbe1ef6c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/3d76bec144d3eef23830247b8a25f5204435ad8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/5b461047ecc20c8efa6f35da2975df0322e6dabd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/c10228a31e3d84384ecbd983d01f4e0d7171eb38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/dabbfe502556d9ec338e0c0cda87e3b33d382aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/e08ba1dc33d547b77f69c9d7467e3f2b4d08266b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823712.21/warc/35910300ea02ef690d1f8cc57bfa2a218b61fa9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823712.21/warc/50c52cc7563d62606c32b0216b260bfc64a53562.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823712.21/warc/9113b4ab88e9f8510208011caf5540e65d8d806b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823712.21/warc/a341309636ffd5fd0828878d78569ff0ecee32d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823712.21/warc/f551e36309f6d215a15a8ce42ac7d22637558eac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/1a3f241c05b78761e2224e56a2931d9a1ac820a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/460bc5df7b867487e6d8317735db7081fe5a2019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/96abeefb2e119cde579c755006811dd8122941f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/cfee61bae5e9f7d0b2872cee239dd76f2bfac442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/dbe964c336eb55692264680e55077c0351aa83d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/ecf703ec366aed0556b48a5796fec4cd67be09c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823785.24/warc/5c8383ea80acede7518dd2bd41daa6cec87971b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823785.24/warc/8b252d148321d3dd344dc8da2acb2925e2ca201e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823785.24/warc/b83bf49236ae27b99d5b031e580e62ffe87dde33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823785.24/warc/c931f04092493b7c979cabec2f1cf4a63b6151ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823785.24/warc/db47b5320592a16bb3fe79dcd7e5e649144b8bc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/1b3017a34dc672e860c4d951566fd1803bc8b7b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/304e22e856e2f5812812b73aae055b5ea44a192d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/85f5f0e45aee7d07522a757608b0b7bd68dd0ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/97385a1e126ea865084e170c28c1d405582ecf46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/b18ed94c0cb295ed4cbaac91b76903a06e8918eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/d5e751b1ed2f0816096e28b1b7c8ffe911760ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/08658b212d242edcde1770a7b17a51bd72a26a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/0fc53fffec0dd650aeaa7b10c327a8bd44a79b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/19a8f779edf7472cbb9f79087e0bde3a69d4e3d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/76bdbe9243087e2da51917c8122bee7d2e8a2466.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/90b875a51f79e0aa05b09cbeecaa6f9f1677ae94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/aeee7ef4b852370553ecae07550f53e70e26c95a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823895.25/warc/355eca41b6d3baeac00c8fa181cc0de1c0c2995e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823895.25/warc/53f28791a14b82aee3e7c72a98041195d682d689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823895.25/warc/83ec101d2518b3c2b84dce8c931b56364bae6bed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823895.25/warc/e683c920c5a5a13589ab6201aea6832bb244f3b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823895.25/warc/ef1ff01b39cb3112fdcc37bf41e34443331ba1c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/16f555988763df32b734eacd867a6327befcdb29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/18ab6859a640d03192a4146ad4f15dcf2112240e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/3816476c13f6815d692caf9e0bbc54bc7b2b3491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/64419ee42839dfbc462b672150b7f6d14659a241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/7d4df69d06e4034dcb7b501c9e083cfa1d618fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/afac030e11b5b504ca15e86d0c587c50290cbc24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824115.18/warc/217fdf7df20956d2594a60dd175a0b33321397ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824115.18/warc/5bbed90f532477dafa112b61c03894afd404180b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824115.18/warc/725b7ee938e6fd5ea7559c19b41481dc5018d554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824115.18/warc/84a516f389e3295c23e63dd92611cef300182a1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824115.18/warc/ae135fc4336afdb08b150a3e2a7a789cbc65fdb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/0cfd2269d687ee9559d41098b9997637f0dead57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/1d37ff4fa4b9eb5610b324c51b7d71d805fbd7ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/22f0c44572134b8e110c3e9f8ae65b32ea8ae35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/77b54dcb9f9b3a5932399be8fa1ec5a01af2ecfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/8ba52fc4a91116c55a2c52f0883d10cf43af9602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/a9571c9d6cc800d11bf2b4ed66928e9a4f030431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/1caeb0d5e222be647a33d213e2e89cfeb6717207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/b03ce1fea26850787249ddf8c22929cc37c7633e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/c3536a8b7f773df701a891b26397f39d19a9f9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/cede9e3cb48d90c91dab4817464dbc738fda91b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/f1a09543dbe10c5e1f3aa8a6f41f7bdd7aa0a493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/f2d8ed53ed40b4f5ea2d8b0cfd9dbc06841bb744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824338.6/warc/362fe3d9d83fcabf676ab7a231c76d7a45bedde4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824338.6/warc/8100f42fafbb230d57578102c95df753ade26941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824338.6/warc/d9ff479a38b27b8099c4657bd245aa86ac2bf528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824338.6/warc/f04bc2f4b20ab751ab858042abfcebc66c91261c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824338.6/warc/f7f2a602c0b46cbcf721924323344adf1b08667b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/42857c5069dc093bdcef76206723963ca8684e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/4a9eb210afba87b12f0045113ae4cf3e1acad556.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/4d469b0c2ba55afeb88cf8cf87cf46c6396c4816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/584b824fa567f6bc13d2eeffae3cdfcbde51c8e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/83819e75452f398b08970e99f7888396265e810d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/8cbecdf132ebf546e80232ef3834b58003071d57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824525.29/warc/05fd87ef4a40a62d5a03a9ac19fdd83bce6fb011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824525.29/warc/1ed7daa0c5452964d1d0443be29748fe6f1e788e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824525.29/warc/46f056e1cfb6ae99dd6d873aa295204a00366008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824525.29/warc/ac9eb0f941436be51107bd5f75d34e4546f23aed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824525.29/warc/da2aac21c45325145c6f90947947bd6aa26c7c12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/28f00b5999530099a694756c0072519496267d71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/2d9561c6e6b6228a731084f44046f755ac22d1ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/3da0aa95e75b5b7fa903c6968316d2f65b6d4724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/4e90d3e760a4921ed26a58535598f02db19e1b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/6bde738777143ff2f2af93b579d56244b98768d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/e272e856e2dbd2edbcc9b97f27a78133bd4bf5ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/1440ef1f840346ee3f0891b5d5b981c119601ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/260c0b8fac44558eacb71447f5057566fd8061bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/339dfc01bb55c6a416b82f7888e16435db8987e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/77bb68d2b7fb060c098bc24998f44954e896077c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/9a507090f2dcbde4107092c36c0130e8fa17bb9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/9dfdfc50c7040ff632b2b5cc61b6d7dd34a51cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824822.41/warc/40f32883197eec830d93f9a9e3c029bdb05a7bf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824822.41/warc/641e68bbe59090e858f64e77f6d8ca7e42ff9b81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824822.41/warc/ad220c5b742bed9564a992de09fd0e19d2994274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824822.41/warc/cb6c4aa7660956b83c83139bab7e8babe8d5e534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824822.41/warc/d8f245de4ab46f187612ca0a0985a33631319af2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/075a791d8e2b524a1f18352599f53cca1be9acfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/5f142e6b03a45af8bf6177c2f32a05685c2bc785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/75dec9fbf6c37842ef9ca71fae1e2559b19a2d31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/8717cc104dc0c659e3106fb9d3985614f22620ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/a6fa73196628b07030593832e8641cde3b7dc096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/ce29050b7742701a18819fa952a48e889ae29970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825029.40/warc/7b6f22629adae7472c33887ff5cae0aa8b91d9e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825029.40/warc/828d1227dab3ecbe3aee6bdd5e235d618293340e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825029.40/warc/93b1bfa10256892169c60c6f9921c56ceb30b69d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825029.40/warc/9c3315ad2df00bcf97fb374dfbd6a38f1d90475a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825029.40/warc/e887d4d2e23eeb7fdadd43d2bb1d6ffafbc93b02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/07e8ec3c20e58c948bc925b26467175f5d2131e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/0a4f4c0ecf25d234a118be6a87e2d968799cc352.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/4ecebfcbde0b11566992633936584dbf8968593b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/b26c5bfd15929d431ce74354e9dee4cbe7309c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/bc553bf63d06170445ddd1b06420a7bafd867b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/e4036c8e35a6e619719c0687051b1fa5732c0360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/16f4fb055028eb9a72176676632a90c69bd9c2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/22756ad21b9588f3885a9b5802e442e7af5b71bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/73498a4e3ae3a59da381c36e912c2743465d0aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/bfa189f058ee7eab76394bdb1f7c2820f7055974.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/ddfcaa3ba82d046042300e84b34f8945de23f914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/ea06f54b56f0d99bd46fd94a5498332744dbda3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825123.5/warc/1cd4f8fad3770f960354e858c2f435f38415128b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825123.5/warc/5174b9308bcee3dcd58cda48c53282baf3d515fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825123.5/warc/79dd4f8c665d1e1593df668f218e7d44db7b3584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825123.5/warc/b946d03e5ac05ba4f1fa2d162e9c884f23454e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825123.5/warc/c235420468a314a6f8352fae0d3d57de8f68d90b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/31b87b0ade9cd7198a9c86a21211e35e52afc24d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/460326d90ec280f9e8d8c0637fe8a8a675d38960.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/46f7a8a5f115881ba3601e3bc6fda96775f9c710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/6587ff8b4fa79e5ed1f91381e9bc0a38d73e5062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/775fbdf8f46392d6ecea79ce0a6ed8fb42c898bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/e8b2b34e0caa9915d8e2981559e6503629700d9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825363.58/warc/3be117d24bb9df40b297602962b8d051bf7c07c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825363.58/warc/8deb541812639b14d79c153162da41cc6deff484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825363.58/warc/91fb71513876afc172a51c9ab629c023e94b5ebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825363.58/warc/b8b2188029b46049604d2c1e0214558a7fcb0151.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825363.58/warc/eaf146291d3369af2413a07fd73f81553db885bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/0af7ea05acaad4cd4c41ff8112e548f2d4db19d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/72c7cd87e27ea4b8934d265993d59ae41b851411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/77172362315bf128d01d0701a0f038656f8ddd54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/8f3ca38f8674c03eb9ae36f98567349c04eb3b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/bf5ccb4491dc2a46fdf5591c74481bf2145f3a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/cd02847636fe0bfd09480a4f97525f118f8cdacc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/1dcbf69b13acc81a13d2263cd31e97def68b52d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/4f1f252b6b50f60bb28874b9aafb6fa60b6729d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/91ea155f4b430830f1b7e72d5e20818fbc3af49a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/a352ae9a12d142b344c14df989f997b5ce59327f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/ecf971b9def56a996add5bc5b5ab5f39a6d0fc1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/fae2c1411bda96354793329480b286fd786c2ce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825728.30/warc/14dd42ed5cc7b40796ce1a2eb7b55218fa224f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825728.30/warc/2e3e21d9fc32381c0bcaa37a0af72adc33e9b378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825728.30/warc/4a2fd196a17586a06521e1db603c2cbb1b68e49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825728.30/warc/60a7375b185693fd56423b35ddfbbc88f3b65740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825728.30/warc/8747d03f6fc83a06b70fd4306d7e12860d52ce2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/5403a2a12ad55c7249881fb7444b0a76121b1121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/6421668569bd697b7ed8b8e81d38f3a23ada01bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/9503378370f1be2376ae3d1dd6abeb0a9b3243d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/b4d1985d20b4017a5ee5081a4459f916b180d5e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/da25d7dc1007b6a367f2f29eab5f0df3e9a143d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/e766a9f07c760f684aa7ab1174e66a3ae360c0b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826145.69/warc/2c5ae4f434a561974bdbdf447af9c63e1d81b11e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826145.69/warc/6c4d0b00451942d63ecf5f3ac0a775ffb54cd45d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826145.69/warc/759f696a485c07d0ef2d1091e90e586d0049a44a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826145.69/warc/dafe99bd076081317e9ce5b1fda8bea25e08e15c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826145.69/warc/e8ae5ad5e33afb1ad29b8f04b4902026b0eeadd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/01af8b6d609576bb0e9b633c3459c0eaff0420af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/10766d134da44786d14849a9001c3267dbd71060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/7b807c0a2153e76518fdeb058eb0a1181307adbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/9d4ff8964e18bd02e09f3e0fc4d80854f5195ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/b62aa4b7f4d37187b151405af695ab5da429d8d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/d52c26b3d09f6e5af3ed939942b98270d1e2ceb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/89bef972fc5f4cf60c28c4fdf3c31359d250801f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/a47cd31a692893db54255a4ea8ac4d3e67d4f8ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/af34b503c085dbe3d3bacdfcde263c5494bdfd39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/ceddb1a572ff5e5b2de31f195365e9f406d320a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/dbafbb159abfbd8422558845275ef2b84070fb58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/dc47d5368bacb59779bf51ffe10f5976a644ab6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826530.72/warc/2b0b67cea458519b4d6b58d5605ddf472e39e97f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826530.72/warc/8c5c173b5c012dc3c218a4dd44c20b6ac8bcfbc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826530.72/warc/c20c73dc11e99372cffe8133e9d4b375f4fe529c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826530.72/warc/d045e0997dba1f2e8e9c562046853355306ae8ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826530.72/warc/f34eca3425e383f8fbfecf299cd1a67525df10a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/23d5cd9bf9b4fb13aeae9e9b6c1d012ddcf14937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/2e09a56d19027ae50a85b72982fb93c3c1c5b103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/45344f3b7cd0d2f74c5b13b6abbcd1cad876a5a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/466a0b66775bca4190724118dfc1653f3670383b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/73408ad06754d4a67d433079eaa67a78a9b65b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/783dfddff23c4a3deaac70f6723fb8d0cd78ab9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826715.45/warc/39162d2f7b3db84f38ca17b56b129b574d1d0da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826715.45/warc/395e7a719ba287d677fe6bcbf3c98448fa88874e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826715.45/warc/3e9506fb8107fdc8acc45f610401484b1dad75f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826715.45/warc/767761976e2452a8e0080ce721b04a43a77bf410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826715.45/warc/89512a84e1993be86175d4bf50252362de6bc561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/29eb1070a5bda13eeab77dcfd3fee9ec6be1fef3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/80f21218ce1191b7111e0531c7d77b59366d0614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/eacf4eef0e7bb759e56e62250a48eacfdaa8f36c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/f36fd5a924ff6249ad86f631d56ff4e2749dc5ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/f7595cc21930453601cf208aa6e1c8eed1912c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/fd4fd558fc935920af650fbcead4543f71be4fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/12c4f4995ded3d05a32a3dc253654f8cd7877b68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/14afac7094cb71e4ba8e76b5d1edb5a56d8db8d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/6c4672124475ec4020b1cff0d5487e77f68621c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/80f2f0665365a79341d96a9c6d6d3eee842b3b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/885b50c24d20a1efbc0de32a5f9e19c6d131ce5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/b02359419426062f8698f2225878613221bb5bcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.55/warc/01b3356388ca65bae5f52424cef6aed7ff9bb3f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.55/warc/5e5515cc11b3742c6912e504d0ad7759c51956d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.55/warc/bc434530808511f8ce9d9847e5a3ee093981abd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.55/warc/e118b77df87d3275f7d7c6d694a3fb32ceb2e143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.55/warc/f46a43686b146cc62d7fe93ee47e58238bc3cd97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/01c1cf7613ca0c85c2c6f9cc53697e4a4af94d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/3f1c84421181a52288ed5fdb371d04dac0392792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/463d00b676aa236ceb29d18195391e373c2fa56e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/acf8f64e8df53b881e17d0108ab3be3e4841815a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/b16ec9925b3163280f37720f7d9bde0f7ba027a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/cfa5f67c48e8e6e27aaec76ac9fab72740ce511d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826892.78/warc/09b2c9a6230b5124c3bd96d16a8a463d0ddce9b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826892.78/warc/146f5803ad15865e565989b049971e8d599e4ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826892.78/warc/48f9583f673898cc2c253e5832fe92c5e427bab2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826892.78/warc/5ae535f9490afe34810d2771b156ce9bbfa5c739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826892.78/warc/96828b3f56ff79bd4e808f0ae3ed18aab0d89283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/1f1f9076ad166ee7afcff79186821afd8d820c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/49c3eed1a3497cbba41f92d38f5da2e51e224c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/91430f563845457c89f6684669c6ca2ec26da5cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/9b1a99163a1163e8b3bb54484d699782a31354d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/a7808f24b3dd21f1d83112c37026529e2e928ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/aa29df30cd10ae4be09feed4aa77c5a29dececbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/31baf82a0ae9b07dc30541a4097abe17f1942707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/3a3b4bd4919b139fa24088685f967301229cf09c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/4910a2894ae6701a29fed84feac62b7826524c31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/8dd8a1b497f2acfbe419f090e022c7ce537b9757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/d428ca9f1852c6800e613a0a9569c747bc0eb592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/f6f9163e19ac287cfc74f3ea32c7e4bc11350e34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827137.61/warc/07f8a6396ddc4b007e206c9f44acb2f3a7361877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827137.61/warc/0f5c6d03e89fe0bc47e2ec358977f9dc465b508e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827137.61/warc/712e583821445fd373998ef0c90444c6d6a39c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827137.61/warc/7c9e45c0d8ede4d9f5aab1b182af276e4a4a5cc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827137.61/warc/98b2fc51f5fd0279819293d81ee231c5076c32d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/3d4da31087a0079bb90b2639bd6de84cd5a5ac39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/65e2155999680e110f44bb1efa81b41a56e6b7e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/7846026a89399bb71b4227bc1694e1e653d0145b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/cd31a005d727233533f1188c64780d782c992f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/d904d709db8223b01cb55a4ff2fd6152c20a18df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/e24b52120e93b4117d586689ccdb78b4bd63e76a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827252.87/warc/9643a0ddbae4a2c1d47823f4bb1a3754f619d2b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827252.87/warc/9e454b4105d9cbc885328c643a8755d67147ac08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827252.87/warc/a43838110de5980b1886ac2573c91c1f0119e3cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827252.87/warc/c17428a6499da48e578af59fa3bc9c38cb0122ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827252.87/warc/e428369a0f00523eb6cc02924cfcf8ecd1c2d321.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/1afd1f75d6d60796acd3287d0791dd7f511169c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/47bfe15262c023cb511102de79c9323ba6fa71fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/6a5d16e72bec0417d36b6e1fa1fddd750185596e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/78e77885e04274ebed487d8d26325909bdeaf265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/c893d99bdbe1f60a6edbc39871f89fc0924679ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/ddf854dbb7d30c2e0c3910c9c9330129e11ad3be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/0e8c8835dd447c9df097e67b4a7c83bd399f2253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/4f53c96f637077a5c7c3deed4b217fcbd8f7972f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/818b26340e991f327719e75b19a33b7bb4fccdd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/af4de54402b5813fd01a7ffd5366a42500ca8515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/bc2c4ed11b36098a28f0ecd599669f4a9937418d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/e7a4296e25c9d89ceadaae94ac7660d43b6c80dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827639.67/warc/244725af6119b330da53979cb9968d119608b283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827639.67/warc/2da3a8c61b3523cff5859882b99e768ef77639e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827639.67/warc/4291b2045233e455f2c9a23f81346a4c72bf71f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827639.67/warc/575bdc02082d75ed2803f599ef154dd0a82daac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827639.67/warc/8a74e71288c2a5991d6f9887e00704d3eac831b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/9eeade3fc65650c914213fa6b61aff8d86f88907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/a0232db697094af95bb72b2fe644c882b792e8a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/a21e63ed78a2817d1959433fcc6449b4f23d5c3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/bb391c476828cdf5a5118f358afc8c410e7a1e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/bee9ad53c0b8cb0bdf68b75909a6919b5855b778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/d31a0bfb8a186960c680dac362edf11c3db89ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827769.75/warc/728030ae5f05601db6425d30142a587cdc0d74f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827769.75/warc/8997c7bd65229ad7ee5283a16fd5b773893fcb9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827769.75/warc/ce23b3d5524559246dd23d83e1eba20b81b795a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827769.75/warc/e1a8ae2394e9e1d8d870ee35b577701e3e0a0aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827769.75/warc/ee910c8edaacc8c4171ac0dd5c5b16e3a9143e64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/6b4158a3b4eb81f745e00676ed54823e4a0778a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/c19cfa4d47df6da049d2e126f7e32a7d968857aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/ce19f3ad8757a50a1fc7b3d02e97fa16ec746ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/d1d11a34ba3bef6e23c82525d9b7fd9449562398.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/d2624ef4e1607558cf99bf6d681a17f940124e65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/da631645df2a609f909b8d2ff47824973ab2faa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/3281a3886349d9961d494c9493ff7a973e61b0be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/4798e322e611586c9c622b41d455ef767b677f68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/51dc715e1dca65eb0f1a0fe7b747a06d1fe0d23c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/ca82da62a364e0d2871d26dd76f7b6349c2431db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/ce5f75ec746938cc433230023f837396c4f9f847.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/d8cc90b73c6c00e0af1aed691ed482e1c3ce3a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827998.66/warc/4047cf814d7546c443ff225c64d9fe3c4999b058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827998.66/warc/54361080e7d33428138d72a6b05a01905b4cd607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827998.66/warc/8c34da8e8b0dc5490f1bed23ed64f546d9602c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827998.66/warc/a00c3e23d4bfc5ae2cd51e31beba38f43bcd8a5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827998.66/warc/ba1943503d20b135b38d556f56bf5458a554d8e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/206de0f4bd829bddd26a00c1b6ce74142c6a1a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/35f0a1dd22a1d9e2d201bd444dae7cb06f39ae35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/4a40240dddde2ce20d84e2d719e931a9ef6c299e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/538b13f1a7ef7d4953f7d008e1e3b92e1333130d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/874a707f85f5d55e571a8f7e8fe0de2363ecc52c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/bcc482986cdcb59d58b6151313d1342b6368de12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828056.99/warc/2faf5abb09f5c69a83bacf00214bbda0d436e837.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828056.99/warc/4e2fba66e1d4aaa3c6a313e3f45ff5ecec04046b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828056.99/warc/83f6fcac735df41488e2f5770c7efc31d70b6fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828056.99/warc/c3f80c655bfb425f2e2b734ce4c6ec00753e0445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828056.99/warc/cb4bcd89127a20907d443e29d00ef9f2a2bd3a2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/12765419589638dd809545c3da6a6ffed8848c1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/1c01bc3eb693f15dc8cc4270606d80be53296c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/27edc2531cdc6b0ac9f62ce28fdeeb29e57a09f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/97eae660f59a535c17ea75d8cee4b92add2b89ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/9b7fbf456fe6906708474f09e504ff635232c64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/be98577c5bada97119fa848858ebeb545d43c937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/03024d501fc301f9b261fc41cca1d7667044a107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/44600ff9e66e62944e5c36a3832bfad919a06e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/49706c4057317494321f18fd2a37d11ba5c16cec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/4b33f936566b679efca81ac3de75120b6127cb45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/60d53dbb424792ab842d647dd89a25646749a764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/bc0918dbd7508351e41e3e7cd357114a4cf89e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828501.85/warc/6f0b86b0761dfe6b5b0d9726107f30889ef85daf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828501.85/warc/75b391a40b4192546e9ee6f2e8fe6f1e7de578c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828501.85/warc/b183bda4fc31c98973e2591f9f10c94cc706ef8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828501.85/warc/d897e4b3306b3a31333a0ff6e8d3777301fe7704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828501.85/warc/e4a7c624cbe6c782f37e0d27a9eb28a853bf8ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/3305319d414a567cbe98086753ada6d00122a57b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/4c4ddd9b7434a337e80e09c6361d20c6bd64b55d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/5e527419a8bd88418f3b4407bf5babf2e5861c5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/6d50eda9403a6c374cb6c441e44fed092a48cab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/70f9b11b0f15b72b2856207c5b324b948a1eb379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/fd42dc9bb2ef3e704046975029d84e207dd9a0ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.84/warc/18c4bf86808ac93093fbeefb84117858b6048927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.84/warc/9dd0dc4c88424549442f704d1160057c1584afb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.84/warc/b47a57934ef8b56da9995d7f936aa4483d696c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.84/warc/eb79b20f170e79f4acbe3f4a8120ef989790c657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.84/warc/f461da6267c729821034becf90ea27ad8808de68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/07651299e576227fbef6d2dcbbe0f3e7776bf4c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/0ccc21b6304a707e52c228ce3779dc9a981a3077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/3b207be08e02c69f399f580a9d5c7a5df31d4591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/cafc986a158bcd3f7142035024c6e4ee29dd4233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/eb676fecf16eb73857f2aae1c6daba587a59d816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/f6e2424783f47896fb27fc5ccc1d5aff3f7302fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/1ae9e41d1c7f194def40ac0b46cd1e960f325761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/20511905db4099361f7cea13fca751271b235b9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/320e12deadf28b5ec03b47b5099765feae75f737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/7ba1813d40657420bca3fb76294bec30ceaf00e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/d04d64db307d13edd6cc25021787494ce5188b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/fd115b3ab3864b9c4a70eb5ca2f415041003020a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829140.81/warc/0061e015c4a4688c19e8abf35133a3e35acc90b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829140.81/warc/45a217a2b702a3436dba4727f2fc825a1eda3ac8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829140.81/warc/4ca398fd4024f906177a6efb2987020d781ac302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829140.81/warc/7387edefe571885f32c54d2a190e54283d7e6125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829140.81/warc/db8fb25af44de9c6165ec8142610cf0eac996c62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/2a32484a54a138841906b00f6194281cc4341486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/4df2bb00006e6f10d2850a83b0c5d9a78606494d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/91bc5d5f487bb84cd877d17d92b3c1bdba42c67d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/c35d10303c44fe603fe6dfaa5801b8be49cb41bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/e509857bf4cb2c00247810a6971c8036ed2cd439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/eba93e1783e143578c1cb74fb4bb4377d7266327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829429.94/warc/0b0e78c59ae2d00c41661b315fcda10142603c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829429.94/warc/2d557554ff96e294bb955c055fda6e3949ed0273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829429.94/warc/3c547e41d52b1182eb4c0faf3e99efc93e78a3d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829429.94/warc/5b59660783612452470588fce5c179089b76210a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829429.94/warc/9550e0739545586c19b5754b23a5aa1dd17908bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/0996b61a16f67f928b57c7b7c12c6e2d6ed97228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/2c35c6e3e5342a3bd727fe52a0e31444761e6e6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/348ca7409122beaabdc93845c3970e51a5c2c32e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/496b01f3634c1a89d5fbfbbc53dccbf049cfe895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/605e40c1f6e5e96fb598d351bd155843dd6be407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/d2dd86fdfcc39eb999b050b4050889501f354730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/3f498d2c33c7982085635ae895d0852691299021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/535a24da989a34eb4479cf85b3eb20bd2cc944d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/6f142a66322ad2f0c53422960b8cc31cc58d1318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/bd2e0fdcb121f5b6da7495a92d43a475370f18b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/c21422ffefd15df8a51fd941ebdf8727f1502f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/e77bacca6e82bea8a68b072f1398377888e479f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829812.88/warc/0c6954cb0b2fbf3d328d1c4ac08e4cfe8c6c29c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829812.88/warc/320d05fc05d9a8fb074002889d9465b6bd94152f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829812.88/warc/bf54f76b008f7e2f76f4136a0c980aed7c79f954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829812.88/warc/c7653781fd2e66ec2a2c9512d0f14b39ffd27313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829812.88/warc/cfdd4aa13103ebb50e97f856a290493e7cb3da5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/0fd4f55c2f3f384dda23d92bdabe29c4ed7f0a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/1dd27f120a3448322f8aede71bfcaa8c7398f407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/9aef3e7401a2508933e2bd2131002d5947e0a41b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/a0684c0ef3d5c72d83e6c099bb75bcf6edbf0427.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/b2ebb6de39ecefb8f6908008d75405cf1f62c61a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/b941e7818c63484a0dc3e0986dea12f3dfab8a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830305.92/warc/072498971afde526685193f285eff687438b0ec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830305.92/warc/1dfb45ebd9ca49ec069d337d61d23d28cebb165b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830305.92/warc/203e9310f75e9269a30fab5f4ee5e07e78a3b4d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830305.92/warc/5e4c68f7593274e6aa738156df4d12f8ded62057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830305.92/warc/88c37f217507715d34a721ccd7c09f5e119d2e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/1049d748bc512a58cddb32ab447fbd0fa3b18208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/411e08b315228fd8fbcfedc6e3daf966253dffd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/43c8fb520338c4f436481a0fe4c30b847793e426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/90842f90453409f460ede0b31c98a7ac8325c824.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/9d2d2dc31afe131e4ef1ddd907dc3181a30d0283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/e41ad7bb30946e1318377fdfee52fea0f5e4c289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/1bd7e351dddc932de88766cdeae4d720bc295d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/45d795943c5adea0e753e65ae0196663cf56aa4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/860a2df1330ab3f2b6e2485a4f8c832bdcb09269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/a6a916daeb3c893ca53ecf668cbe68ea69846635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/be6ba92c52abd59e00388e0bd89608e3b5a4d3c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/dc0eca8afb6b0904ac6a47334e385f69dff7990f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831715.98/warc/13175cca8f1d021a47fa2d868c3914b34f4eb412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831715.98/warc/76b86ad7cbffe3f8facd2a95b75512f410e98284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831715.98/warc/c79794a2bc582999b8a9a04e3d90688495493b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831715.98/warc/db0860bc6fe3cd22411c31399a47c021f3f4734a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831715.98/warc/e805669c5818db62c92d69d3d05635d7ac4ca862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/21d2cf2b731cf1ee7a7a13bc91dc46ec940b2269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/2a31cbcb6c6e63ccc2804c090582544833f07a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/636bcb3d77b64704f1a1906e322869d239a4de67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/8ae107db273d8d84736dedca5550da9b05a3a66a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/8ec44911d1bd5df4ee7a267b2df777e5946768e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/973479c6b6582792ed5517b148b6d6476385e160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832259.90/warc/0a96c43e31446ab557f55a83cc0647b21dd6546b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832259.90/warc/1e7f1eee040756288d5c1a4ce3812e0c794664f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832259.90/warc/295f8086eb729e7844125f3fff3ee68e1ac24f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832259.90/warc/3817509c5c915a57b11c84d5ddbe8cfd3c291148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832259.90/warc/4a45fc05c18d18eca46d8942cb0b8afd7d8bc49b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/166ae0e960966737763f0b360f80afd1f9968a84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/35f3a0859bd4ffd0a43c871524c6af95279b7555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/3e7a7b3c3e0f9517bea344ee33add4a8a4113a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/45ea03b599dcb23990f44621abb00c8c410f5730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/76640e57f6385c0529875b9ad6ee7ac6520f2ed2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/a533e969b9114f7f35e707d02c302e77d5fa245a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/112539fce61177d94fc7a225c68d6f401267d7af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/2bb56057a1751f81d7dcd26fc0dedc5a19859044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/414e63d24dacb9e8ac6473e5b8cbe34978c47a10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/6c78753978f540d61f23c129e54aefd26e97f338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/75a9857d627e033792e064ff278170aaf1003cff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/8159c9a764dfd50679d9f275573ec3993c4f140c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/debec76df9a43cec2be4d24aab0a3af4abc5ae3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/17b418561a9382b55d8f8755c70840e9e4ff4e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1f74d5de5d3e5b6e47c43b3431c9d1de324e0ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/23e784b57e42afdc242d3bda8e9e73b34a705524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/25e2ba0f6daa57ee45a37f24d98b8d9ae061ee75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/2611b18821a83588d036b39a3b3c0fe852ea8ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/2667c15a2887c2fd047f27f9d8abfa65f956cf49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/26f4b469e49a3dbafcc0b3eb15d2cf5cb76826fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/2a6d0acae041f29ccb9a25fb20e09dadb57dfc11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/2cdce88ff11381221fcfc943e582baf22aa90058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/2da86b99884b370990a21dc5d765a02d501e3e3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/3343edf5f76cbc56c50cb10687f7bccf2aea3d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/364abbd164047fd2886ab5b3ba86694cd09ff38a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/3f55d3a34aa98fe7e698d679a88548c81887316a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/426d70b774247276e31d0cac0adeeee35991570f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/43f00ecdf788aa943fc79e0c8db36932487b571c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/462ed455e4fc128bcfe75ec3eeb3f9686834d0b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/46beeda96746ecc5473278cf923852294dfa465a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/46ecc6e0da08ced64c3129bda48207729ab9c422.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/4d687cf2b614008abc973daa3d75f58ffacb189b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5292dd4447a32dd836c28989d0991c24222fe67b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5453d1113a01feda0ee28512e672bc420c4fbec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5a262d4a8f3d8ab8b2f0427ee1b9e828e5ac6019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5a70f8bc64df68935b36e2f37716839051fb369e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5b0249f53ad56d64a86359cf7a6e9e37d4e07354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5c74be0b1a0b8ef2dcc5a9ff94773dff43a02210.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5cbc4ebf82bdade6c0fcc7941f22bd792c2c5353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/68708b190a6acf783a96ba8c8cd1d2c70219981e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/6b5e9bd32cb645fbdf09c012d7c159f785b5709d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/6fc8c30059dddbc3f430cb5901653c01e704fb72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/737c9cd69fae15113c37d347dbc168dfa9eb12a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/75ccc40a4c271b1ccffb8796c308278b33fae255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/7741d848ce70b89d9dd23ffa480ce442cdd71ef3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/79c6d11e712adfc5e20acd0f52d8795c4674ccb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/7fa841ab3ae6dd2fbfd0a92bd7fefc91ad571aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/823df1d5afb4a5ca5e027e2861e3580709d3759a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/868ff774999d0b0c7fea5a265c86cdb4959c6380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/91fadfa3fb094e99e5f87d77e5f9d9ba318b4b2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/94daaa1783f2567763785485675a130808659dce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/95b416276ac8494583ff1cca06dfe84ab082f9dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/9bac4e6fe5613ad05770dcf84eefd3b7466e6209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a2ec4d23ede21c1d6c48947cc33889baef7dd465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a33a16ce17f2b21d4dc4f26d23343e71dbacf8bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a3db9902f1cd92e5e7000a4d961dd9dce79b875e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a4944d47e396862afd9ac474a32c0b156771e4de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a4adf1120ec3c9185d11b457694264f4acb21b4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a4b9a2ea11671d8ed690fa39450d9b1a4cfd9f50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/aab855962727ff94d67d894bb18854ab403713fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/b18d1745ed42988d284a9054ed92efd172f59162.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/b4b35dab1c7bdc020278be729e1e5213a2b5b62b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/b82caaddd835ec2022f16c3d88abc5efac505361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c02ed8df7c1f3adbbad1af34f9607990826358d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c0372a00ccb98668e46dd14c605d7e53735dabcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c04befc5853e0bd1f05ef4222dbbbd1dcfcbfc96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c06b1f250f25b5f105a052a71ef6f4c40abcd706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c11677cb29c3eabffa758c8042870583c960a5fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c1318a6047421ac8e09d368244565c10c9d4ecc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c8ede41cd2927a3d943cd0e0283aa589a5a27135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c9bb73033349072356a2a806c9c02523b06bab2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/caa01ea8060c74c77ef6c4ca8b3665dfbcc68857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/cbc9450a652af6a8a8e7e7dc4b69606af0f96b02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/ccda8ed1570919447b4fd6707b33f579eaf55775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/d09733f3cf90e47864b98b9e6342fa4e42fc8ea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/d1c0d3cac8d3ac41230d6f500e949cb2dca0643a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/d6a68b15a98ad4a4c196365ce87c3a8481ee8dd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/dfd73290583ee25927393c177d0fdf02c7057b66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/e987583c76c79ade747fd971443cf4e4cad92a67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/f3fa06636a480703fb01d29832f7d978909ee3db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/fd828f9bfa608c500787897da4fb75768cfbbc79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/fe4df91f4ede80abcce44b19bc213f7257dc7c96.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2018-51/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2018-51/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2018-51/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2019-04.yaml b/configs/cccc/dedupe-url/CC-MAIN-2019-04.yaml new file mode 100644 index 00000000..5a1668fd --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2019-04.yaml @@ -0,0 +1,661 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/01d1fd8ea49130deabc4d3ba0248f0c40cb7137d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/06442b418d89ba606baf453e96cdd0c0a8420ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/084ac20d7250427483f3ff6deae5be63597a648f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/119d3cacd3d2633fe9254fb5f5fbe89c5c6c0836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/121a8edf7bd6754dfd85d0b38a2b242e0fe9eb37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/49368059255e877db2f59f2a8a4ab9bcd26ca256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/726cbff98549e0a3c0b7cefa3a58a97d010544b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/76773d9b1e3bce965d187e6aea9f48cd2eaebf74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/b9301b4fce7253cdfad9db6faa816321e6afe8fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/d76de6c3393dfd4675b736ebb588855a9f7c0817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/f896137b71b4571ed3a928f79429cae6ddde4f43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656577.40/warc/19191e61b30767420f88e7392f861ba02ed64e7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656577.40/warc/572233b6ecddbb68aea63978d1f52a54561d7030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656577.40/warc/b43698935cedb531c068ea12ac457dda6018b97a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656577.40/warc/b4a6acbe9198e75fbf45ac40793e159021252903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656577.40/warc/f9748bf98d8ea4ea83312c9064b6639b5f006934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/213e80975c7f7574dfc7279dca3ceb4fcc4af591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/2819d31c544d761fb9250acc8e5395e42fd19702.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/2c3078d139f53714008bf31ca7d54ab055a75337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/54939dad447e362a87828b1f1c8ce2d20a6ebed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/7cc58132b01d919acd653b9bd06dcf84a44b3725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/e6fc3097abaaaad6888555e70ab3c915a72bcbf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656897.10/warc/6a3b977231b1d7820bce9ec4ad4804a515be2506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656897.10/warc/7861ecd3b319c647d11e42223d884a7c974f3cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656897.10/warc/8dcdc98722d768c88d99fcf0f0de360976321a64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656897.10/warc/8de6a9b763d090faf5b3a60df1d5c144f248a490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656897.10/warc/d42340b587a1d5cdf192fda81310fd1e91f9703f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/5d9ed56af027f2ecf93fa3410c237cce6e0640a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/9cc41ef0eb8a1a9dde5409ae982ed927217eb1f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/ba03547b0b72e4ddea06b577c93fc4122318fa51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/cd24c62e69aa11dd8658e7c66c9ff57d3bffdba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/e810a96d592b3cc1f4c656efb2c6ea1db6915d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/e96c10da1261b8bca0d28d2ac377d33957a7ee1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/0305fe848292bd786efa650a9b08eeb4aff6d970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/0a6ed93587aa31644e6428b5ef1f4d902fba4eec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/0adcf81cc589362af6e89c213419c540296d70fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/2f8e47e61d63ea5c5596c08f44fd3dea5f2d8aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/9d19dff5b8f34e7927e8723f9c72f89dcf6e8eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/e40cffc6b258be63537bcd4d282920030bab6e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657470.23/warc/04edd6282b9ea0e535cf211a8b3833592d03caf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657470.23/warc/60bb9392b391852cd70ddebc78a81e34f25caf23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657470.23/warc/b758aff560d5a5c5db7416be3dcb9bd811becefa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657470.23/warc/be7dc12ffa6a0a09c40e7a515d8d238aaf6133a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657470.23/warc/cdcb30c463296cf97b8dc8e56c8a774d6ccc9fb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/06cc4215407c3e03ce14981697b4a91f881a155f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/1bcc87c3a8083f11a1cab8c45ec71bda69c4ce5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/7a217d7e65cdf1558c4a22a1affca5ea39f47be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/b9f037aa69dee98ef1c31554dfd92d0323170155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/c3337392659b57fa5060eb339adadccf2f91579c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/d366547db674bdc6803472d60e10449bd97e63d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657555.87/warc/54276a381cae1a8fe8445f9f55ad51553f8c1c31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657555.87/warc/55efce6243f244fc0e50d79f7b341f49e5591f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657555.87/warc/b8d2dd2d2f78717e07145072b6f1a8d9345b9c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657555.87/warc/c9c2117fd33e49f367021e23b939f0ad389d1ae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657555.87/warc/e05c56f8b678d7404ef04cf788e2913b2a0d64bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/03d839bfe190177cc04d23e14ca66b82a435f37e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/2c1429347e88a534f9a4486b3dd95894563f5fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/75d1dbfd142d04fc9921bc821196d32e5aeeb52a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/a67ab0200b934e6041a267c4e1fe6f850056b3be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/ceca9134caaa2117a3e6e919964627eef2980dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/fc2637dac7c532ce33492962af9f1e5d600960fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/3340e83100292a5ab44dcda8638e34c45953af50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/3a9838d5e25d0cdbd93eaf5d04fcf8b51c2af559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/698a299b7b2b0287400321833cd4667a82ce4f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/87c79be27fa7a16e6514234bfb1cfd9f0dfda256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/d9f4408d4c3fe383bc0c4e1dc6fcbd9df868ed19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/dba068d51e16499d47dc6a2cdd48667e16b60572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657907.79/warc/58aadf3f3ac84e037e9e91d694b8ffb87a37be29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657907.79/warc/6b381f743d2904bad344b09ee9c0ca43c68e46c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657907.79/warc/86463a910cadd114b93a996918b14c6338159880.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657907.79/warc/ad9e968f60858c685d5e7ca4f429344bb7778284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657907.79/warc/cfebe0e18b1856d1708f85945299fa1cf614d725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/27a05adf1bc8e9e98a9a8f8c489fbc702dd36ddb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/2ae2a8ecd3b828ad3b7ff35c0c2245ae29e7e651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/c66cf903af86f03a4e92c06dbafbe7474904bee6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/d1f8f94d88e9c7337835e35090251a645d3828ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/e7a1f872ba1b3190f86416c33a4f0bb28855559d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/fe025a41dfba5a3c2ec6a9e68da13d07bf7fdec4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658681.7/warc/0dcfb8b1f6f5427a0f7c95fde31aeb9f3db874ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658681.7/warc/19a54f78f03c2f8c3d9b62e2b24a93b7318b013d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658681.7/warc/485589e77255cbaf559daf02c1d52bb010606cc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658681.7/warc/c0d264d85381d5b1c2677fb8ef7cc5449070ac8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658681.7/warc/c8d856d7893022b605347cfe33a551a919c0b09d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/1cbcffaa441d2c1f1df5b595e1d746a8d8b04960.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/39ef1e141d9b12910d9afe780134635dee99e5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/4152777a1bcddbb71db27b46db5a7e75bdb05f46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/476400db3bfccbb767160922285b5ddace58106b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/96b836cca3fef291a3d5129ac3886a669972c622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/a63b989518af0b143e4ac69dc888e786e0b060a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/2971842eefff966ec7a2d034f9373b089bf3e64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/39ff2e9021633dbb5c0233b8c1527bd38c3ee211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/419ea9218fe8579fb89688bc3e15cf1cfcb1210c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/be4b6278eb4fca9c2c9a6a4b0bbec76ad5c21add.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/beade28e99ccc3348523a4cb8c340f14777f6f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/d927e1b1f464f4027ad8f8e49735e57915e5ca4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658901.41/warc/0003e6b18fcbf7173b18567285bd3c2c67f5365c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658901.41/warc/1e85feb1c766a8f721e6fe9b338ae881092a08c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658901.41/warc/2e0d607a5f051576128c0a00d8dace4d63cc54dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658901.41/warc/70500e2c3bd1e3fe9bc2412152e75934825763ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658901.41/warc/b717b9b034a8b1c8803dcfc8f7e361c15db994bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/0bf8ba0613134dc68191e154a0f7f27700309f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/d397bd8342980840626e87d632b43d4463bd1ead.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/de62629df2605fe666acf110af27f0c6ec230909.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/e513cf3b0841a713ba2761231818a60a12756b0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/e7e70a1553a315901984546af19d473526f7bc88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/f389c2e64dc36aa27bdd8586c24307c5caeec5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658981.19/warc/099dd2de27ae4819465b0b10b945af94aa4e241f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658981.19/warc/9fd5c70d3b9519f21ece118bd48994ac83bde74c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658981.19/warc/caf294812f00464887ec9618066f549397eff5df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658981.19/warc/cf692c6c9009e28b4bdf165157f389618219f5ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658981.19/warc/d042e0aeb975ed164fc14ffb8e591fd584e4c88f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/12043f0cb1ce180b7a50f8ff85625a31be0f2140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/86536a683669a301dfbad373a952252e5458465e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/87fa253deb94347d902861ebc3bdfdefb130e716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/9de3e7a54ee973b87c9af414619c7a8ac3d4383d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/d0f4112207ee61889e9aac408b6180ed464cea42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/dd2e7b30172386d84da97a48a12d4776df886bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/1b665af3f981a0b6122d967da48bc7007658e4c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/79fe4128c904959dbca8c29a03535798862bc218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/91accbfdaea0d1d98ee1b1f6d9311fccc016cd05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/97d2297d817615926e3e74f4bbb6a68dd8022f2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/d88526417147882f26877513ab8fe3b57ff9dc58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/f27d2de125a53ae1266d4da6a88a81502e4aeec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659063.33/warc/0d5a4b606824dbd80d45e9ede658c71218653e4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659063.33/warc/1e0aa32b74505438a6e0190217435f25d8776ee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659063.33/warc/604da6985d64f2e153e9a7b1db064e8219b9e73c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659063.33/warc/67200ca692845e6063130e2f4b8206eac00ed0c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659063.33/warc/cda9b3549bfabe5e72a58fed58a695f27bd2100b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/0265806333ff30fb2c59c85bf6cacafdc532e635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/68fdd2766f2567837dd613b5910d163bac66b717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/8b84f8b009bc6eda0ba4465caaeec6709edc7e5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/afafc1e123a5d661dcbe8fef331d1814db978808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/b5e821d9b751f0ec4a69332f2e826cef3918f076.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/f0eca7e7f1431807e8789035af3ca2dfffc9fc27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659417.14/warc/2dc57fd417adf6d0a40455411966ce62a53627a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659417.14/warc/81df551a33ab768b5fdfabf418c56a3f02b5dea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659417.14/warc/954456a91ca033b6f5f3db0771187f10d8f3736c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659417.14/warc/dd0cb0d1151995ed89395e32d4ae1300d084dc2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659417.14/warc/efb614fec35560cf5640b21f7a2a7e72b233a06e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/22eae9990fb6b33737e37c13a07a2d191478b712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/6107bf89e1b2b4600a770fb01e48e59c6d9a3e38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/80e8e015575ebb49507bfa03190a137121712a14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/84922e99a54b69a6c2f37b026f10c1005b2e073b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/9f169af0b9925b267cbe658f5dfc17a82dc9bf0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/e5d5f1aea462e5c2cbe9fa83cfbb043ed821bd40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/3725f00483e52f090138ef9521942900d739109b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/c270401e70de87301336016667638758abb26d81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/c74fa65bcd10322ac9e9834909c4ce04d03e466e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/ed0734823751ca2838263a720f23729a36052db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/f8d681718baa85cb7ff2e6d082b0552cfd443ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/fa972903ca5437a5d532c49b7e404b40f097c790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659890.6/warc/0db74a781c0bcee9925a3f6b49da6069c3d43f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659890.6/warc/3090a7d1c2920dc450ade31dc53e0fa842b7f878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659890.6/warc/60ddc98fa92f483dc917d2fc3d87d34e13c85582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659890.6/warc/c238e60ba161fd4821c073e89d42eba0a89f45bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659890.6/warc/d24591981734c23bf14c8cd681664b10478ef305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/46b6bd92388a8df524d9fbbd3d6080706752e42d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/81de295504df4b0365ae0f6c5e71ac7c29fc0a28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/8e5e3061c628550807954d11028b97dfec16f0a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/9956ebabc1bc96baca3e9e5a569a81af15b4f112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/ae1fa200d16013c139997e06a8ce6e2808e1a73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/f535af07eeb62c65f347df4060fe6afc480fe968.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660020.5/warc/1d32bcfcc37a18928a23000d1292964a637d2075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660020.5/warc/362d130be56b785aa7f316c5ef6fd18ca0b3d280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660020.5/warc/5eec1b6cceef694db7cf614f71c1f20f2bf07a53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660020.5/warc/c44127439b5abac4b4dab809ca6eab4348ac2855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660020.5/warc/f8bdb505e7a6e3f3900802e6c339662ec7799ceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/1391668454baf8bde2be1d701b728baad7679b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/963960bee3bc2a2e2a1933925edf6f9e207d0dbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/b87e63ca925cefe7ceaf06ad1cddb8a2a41b7800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/ba5dd2389ccd2b6de81efeaae5f14936f5da4ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/c90981ad5e7606f8a10d550870e9bcf551d1bb01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/e44691c3c07c6fec9cffb17a83e0f06a441d0a4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/286e5d0bc6db6d888b77c8a3f3d52ce02d28d22e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/2e2c776b97caa99ce757796d2de0895244e0be18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/89c3e0c1bbdd28c0dbb9cd4ae66ffa7a6d8f7b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/924797b6e9b7dce2154a506016fcd50c59f0eff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/a564ef002f5e78311808bc6d4ab2ac6fba9e3961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/e1d57a92ade4868c08190dbfdd6f43e0431055c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660175.18/warc/598f27e06159fee3f530cdab6d962a78d5a80b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660175.18/warc/b9f8ce10a8e48adb060b37a18fd235b25cab49ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660175.18/warc/bfb6ed68b3730a7260f07b678a7bd1120cb023ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660175.18/warc/d6f48c92b8aac04c5f67f443713e7df035e691e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660175.18/warc/e197324f65b546d8a34562cba03490ce94259e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/0f43f7659205dadba3a20da781a9a0ba66c14fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/235c4e88eb6dd843daeb34c3d33bbcb0a3320846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/356de7b155c747b81d4ff896f0c1ba6704c7d2c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/61e0921f16067c787e506966b23492815b600f1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/bb3657f5cbdc2cd63be1f5639190f356ccaafe6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/fd731024694431523ce61351cdf75f2784a115e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660529.12/warc/2517697a35defcd20cc7d91e576ebbf9aef0ba54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660529.12/warc/30b6f2d2927bae359310ff51e2338d0be2dea30f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660529.12/warc/5e31c740535d5efd606835b1f17539a2666ebb48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660529.12/warc/9df0a3c9234525cd6e707dbc49286b83b1148bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660529.12/warc/aad5284a1a8ccf884146cb7ee135e46df218f836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/6246c1c08226e3c13ce68aa1dec8f99d9d440b33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/687e2c9e3e963b107293e2d8afc2d7420d33d8d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/b08d5c900b2174f0d7442e1d61acc8dc3d148f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/b44cd88f9f5fd7ad0ab2086a843532a2265deff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/b8b2936abb10a4271789a3ac0dabf4386106d5da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/c106b6af0f7d846fc410f1ce69b9800014cf5021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/09ad99fe1cfd53dd21c63383e98bc0211a5d3bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/26e56aa895b239a8c8264b20198513a971e2c68e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/434607e9d8e509556ecbe2f94f6606347bb5cea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/ceb42241c949b2b0b71f176e130d9fd9410198ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/d3fa89a6b1037ba753d43969224b1c3bfd60c763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/e99d9eae810818bceb1f91362e3e5d57836f658c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583661083.46/warc/05f579c119b2c7c020c6e1380c99b1bf7646a535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583661083.46/warc/4b4db4e3f77a788a96a14534452f9785a1bf30cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583661083.46/warc/903d2833c2d497687ddeebb50aaf32f6d247b52d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583661083.46/warc/c07b71faee97a59deddf580902d4e9f59c8437dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583661083.46/warc/c256e909a00402d4a5456c66dceee43ddbd12eac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/0d0e1890e8820b84897dd3d7e0702ae50308942e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/1fddf4d5640addd42e3e63889502e8b9d1bb2ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/270078a4fd81ebef185874e6107e03ef1174f974.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/726abe745a7f57731f790738c5806c2226aa8075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/826c752249a38835e5b4b3f75937c89c90d98f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/85412a475b941fb600df0e781a524aa61e80b3ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662690.13/warc/026658916739a90d87f59adae949386ad3119ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662690.13/warc/47100de8df394f0d411fa583c927d468f6897d8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662690.13/warc/489ad14fdb0506bcf3998d720503de3906aabf75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662690.13/warc/85dca64f96767b153de9551bff267ec7821a04b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662690.13/warc/fea82c29237d8a6fc8808b5914c934e29d314352.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/00922220ce19c7742b06e8737c289275faa3a5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/1fd8bab0fda223a2c83341cd6ceb1d16aed99c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/af454d4c4ca644c1a5edde818d869225f9304602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/b1073d93ae56c176f998d2368eb5e04b22169efe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/cec9092b5379a6cf39cf8ecd0af25852efd55dd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/d645c65495146b392a24253b62019798a8c9a1d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/78a4838980463158a5b0cb1f7c5857f8906bf4b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/7a4541296eb8ba7bd8005e97bce813b1a1ecce25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/e1bff4ad2c2072bdc5f5c50586efb06af78cf730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/f62f32db58fa4cfadd2c1aeedce822659da1ee38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/fab039cb53fbcfd4691e87561f6dda1eff386c5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/fca73432583b171f30256ffcfb635214affc66e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583667907.49/warc/60609a30652826b6431530f4aea776e6e5429cd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583667907.49/warc/6e6d8a7983d15f0d4da0fc0142f4a0c9951ed2a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583667907.49/warc/743eca286ea7c92c4ba2eff1742f678ef53df5e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583667907.49/warc/8fefc0caa50c20c9e6d96ecf2a277e98e32ab79e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583667907.49/warc/b3d90cd7235b146e61eaf7423bbbfaa625bbca99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/2985d084274d0dd605785d29805148530965d0a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/3c9fd48367ffae71fd879e1ef36d66ed398f7221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/41f612a1d10e2cab3afe929a439bd74604a3bdf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/6367c79577f48bca005152b80e26a01c32231907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/db43bc4bdd602d37456b2c58fb6e3d68e25a30f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/dde5a860f723a1e4a5889ab465ec521fe8a5e285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583671342.16/warc/5a247058303274f7ad9e29840c05edf9e24c4fa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583671342.16/warc/7c522401539526523a5c4e7138baed72ce662486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583671342.16/warc/83ec9a04ada69e6cae22868f15f5e4e3653bcf05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583671342.16/warc/b210d875daabb7286a610ca80bec83c35d625871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583671342.16/warc/ce2fbd78207e7f8b98f886c45d457aa26378ff5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/09a5f12542e5880dbbcd9e4c0d6275a560e3b9e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/27b1eceff9800f36e757b6efca294139a7866afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/37c8a877e64ff39ca26f212e6265b62278fb5c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/49847de3cfcd64dc1469842abfa51fe36f069179.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/52934d3067fcb0a9c33159b9b3a669a24098a113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/ce2f1fe516972e8f2249b4ad9d4413ede6ab30ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/3bc70e198f77de92b6411a37b74d5e48c8d78c3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/3cbb69adf6a5f05ab05ddbd409a1118ba6d23e3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/60626d81dba474ce31be7613523ea4966c1abdea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/673527ea7c4a90b26dfd18fde82ed91f50e8bc83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/cb8f473be5a3e00210a750ced593c9141f6e1ed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/dc4b9a8f4c4884b09324935b400c57d5bad53589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583684033.26/warc/4a96c60d9cd9728c059669615d76d777591d7900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583684033.26/warc/5753ae742d1ca6938f5af177645ce4b254c0fa8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583684033.26/warc/c081cc845c13471399c8985e3e7925cfaa0d3e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583684033.26/warc/c322befcfd3d212b1282606d188d3fc091d41cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583684033.26/warc/fda664b5aa9f121f55dbc15d1d5ea5fc60f1b83b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/386b7348a3b4163e440327852897a866fc5c2759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/5bba8b1d24112f7f5bfe7d7842b5207ceaae5ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/a7247e59f91e90bcf7ba256ae477b9b09fd2d58e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/a7c21ac056239145ed404768cc8e64c72d8dc77b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/ca41050cfb449a04727ef957c7ccefb898fcdbdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/cac65a023f49abbd9d88a2fca50ed9141cd4d968.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583690495.59/warc/09a5a430f91dac21d6c269814b597de9be189678.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583690495.59/warc/6ccb48e35ccf564df35c902aa7c6a819918459de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583690495.59/warc/9fb46371e63e9e01e17b4aa3c50593629d4e0b46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583690495.59/warc/a8a196ccddefd0d3cefefd665976ce0c7c1b0da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583690495.59/warc/fb85d26b2b089b7f17353fd7e4cf4233ea260b14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/0d2de644eb1793a10c7aec9e07b5ffacea72e0b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/3d34e674c9724a7cca805edf20b462b18d588c71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/5f7d538e8127232f6e1c1dc9b04035a9bf28f85d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/dab9f8b9d36e4e37dcfe2d7ac964480ce6df8662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/dc2add068da9154b9579a62bf173515237635753.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/f2c10463215081f7375034d3755ae0f386310545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/5dbb6e2a99d47e75b1bcee84354cc00e74217aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/66b4bf796d610006616d6cda8a49ce2557bf59d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/70cf8886c19b7d3ccb503c2cc1332f4b5b47131f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/bacefdcd122573f72752a69e8932513be2ffd879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/db4f162a040d3c4c2051b17b9ab5e76945e53b51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/df93e3b8e86b1d6577299cc6a9deb1314560d78b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705091.62/warc/a6e940ee18ec1f0c3f00a9984f3d426ec44d1a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705091.62/warc/ce82880a98c4f57582492fadbb653f88dd58ed1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705091.62/warc/d6695289c196589b760e3bf0d6907036bc297182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705091.62/warc/dab2263cd40144a50bf3c227f940dd325dd05b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705091.62/warc/e3a38adec56850ddb6f7a15de7d5c7f6f01ce551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/16b8fd6413ba7c98754e14d9c1413c04ce399213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/23217d8c2d49f9978d50d0f4e48011ef0bd63e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/2df13083db1b9088d96599c918a97a1ee6c51cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/56186aec3b96f1c73352ba02798f9898aa3770b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/6c7d22d644e73d936517b192f19e13c8921b96fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/fe9c52bb496b1033512d063cf6cef02c80b96ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583716358.66/warc/23f2d3d57b971d26f288b274b1506784183830d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583716358.66/warc/3bf0103b823b9c80922992edda5136b93d99a948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583716358.66/warc/9083c96d848c640279f2de07aed42c630d440e73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583716358.66/warc/c48557c66d1c987a147133fc68a0f78bee486a7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583716358.66/warc/ec8c04a23cc7c407deabddae19dd9ab94120fa4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/12c828b6175f9546f2303ec4110e0802199b00b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/266de9dcd0d2064970527c136a6c1829bc2b2d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/411e8ea920cb457d5bea72b160b5becb01fca565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/4abc7e05ea4be81fdf2af0f7f93fdd29d94942da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/821b0bf9cb0cd8eaa9ab3831bc00c91f3ff64aac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/98a534a30830524ea9beed6c5c2e490bca4edf53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/290b50af0f1be73e13d1b978e666d66f37717e29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/794f27c5d86ee384d8c1cca5d5f44617ac44fc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/9fceb8a237566707e656138e61672d286ecfcf5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/e6a18891915338c9ace33c87e073a631a382212d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/eb0c8f3b0868d65e944954b3588a022c8a51155e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/f379969cb2928489a811747e9215174a96b5b1ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583730728.68/warc/7a964e7e069e8c02bbde2dd3fc0a2e425a1cb317.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583730728.68/warc/9c8c04908b7de07c374bed570efc935a1f71fffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583730728.68/warc/f872a68880b916dae7ed6732ed7b01410b217f9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583730728.68/warc/fc324ebe99e714a2abe01f4c7fcada6374974228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583730728.68/warc/fed32931b7ca4b9c31cabb8b466c9889f859097f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/301c7bae802cf1c54bfc3e00d208594c4f8c7d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/6b25e3663beda7ee6076a39df3478814b89a24b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/8a1ab2faa31ff62260bdce7cf1f194841d2b03f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/b1535771149daf60fbb0bcf5cc3fd6044e1d3a9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/c5288959e64344e13cde10ec321cddd452167c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/ec938b5a974777917358b91abe1effeeb7152d40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583743003.91/warc/51ed3eb3046e869fdc4c875affa2eaa88d817771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583743003.91/warc/78a002b7472f3158a4f35231fd326053d5647e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583743003.91/warc/9bd1e657ece86a763dc86ee794caedc909087d02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583743003.91/warc/ab9a81056e20b1c26a73034fce8d8ce6ae8f4db4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583743003.91/warc/e36170806918b56e39aa2da1dfb5423a8b3889c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/15df9c5f01b13a1ef0e8ad267515c37814250a39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/1718d0dc2ce8423edd2703020b650ad91f1ed8f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/1a4ae9539794064bc6e5eaf4673d9bbe7d69a29b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/6a596c54bfc32b44ceb5014f63ab2777063668ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/bcc3bd7dc3f573e57bcf3b77aa56d9674d6ef9fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/c580e5ed4937238ac26a76338465e09e4daefdc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/1064961f3de200933e976520cc830ba5692f522f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/2298b1af1dee278ccca64d3380070447e1efdc45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/c9001118276b8208e6b44addcdb62e39f6580a30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/cda6f0ed541fac0f701ca9af80d2ba5f4d618a68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/dd644ac8954af99f0e59fe5ffecc54d8a32a16f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/f7d5a8dcb2187c814769c22e29c2947f6910cbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763149.45/warc/29cf92ca5659587e92b7009f0d03f0bda9dd9389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763149.45/warc/3deec63329316fe21f14eb25b275d28ff7623cce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763149.45/warc/473a959857146e39a1a0467f051600df74634a3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763149.45/warc/7da4dfb02dbeec9ed392e74273f6cac10bd16a79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763149.45/warc/a8f7b119c6e751d7a985837c6ddaf0a68d204f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/35176be1ef05ec78ad5ebd7598dace28150ab1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/5730ef254614160a1a26c9a43720a9c3b2a9f50a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/728715102b512833f648593e07eb1844f3dc03dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/8ed3c63fd7325f929df6041824b54640566298d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/ba7831ae56f00dac12960ea3285e27dd115b4a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/f162381a1a8d5e286608c6c6e3b13df879b42e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583771929.47/warc/058d759860813f2616cbdd62404aa4a705992da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583771929.47/warc/5179a72f57927299e2160953d77a2feff17b45ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583771929.47/warc/76858f79e421c8b2bce9568ab01b05407228679d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583771929.47/warc/814ca3956f950484c263ca0c526f8898bb553859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583771929.47/warc/eb787393a006ac86d55cec1a6423ba91e34fccd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/2ffeaa68c2d385953b02e9ae6077a8ad3ef1af34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/3934a0a68db0535d1ab00c123bba3b863dae2a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/3ba35a5b922e283e5cbef4518661500adf7321ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/585f4be6517527c679f258022a04b8a869dfd3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/cbb899c3bde8364939497a4c9ffb36aaa71b7ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/cfc1feb05eab5a3cd67d8652e48eec14ce4a0e8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/00613c1eb7c6098b354aea5b08bd7837bf700334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/5b34d218f0725d8fa981409263897ebc3361e373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/7569366e5a4e5a09a743f53c2ffeacbdd9c6cd8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/75eeb840fe51e1489355d54770b99a0f29fb34a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/9604d2e2677cd6049b81d62ee338b5bbe25835c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/b1b54d6021611748b849300381d9006ba20961e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583795042.29/warc/03f6d4c7f85fd7d37db6a2316ad601809b60017e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583795042.29/warc/0c6b87d9b69aa174d9bef2515a470161edfeea1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583795042.29/warc/a4b25e78561c8fd7b3f15865081d1bdeab09f2f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583795042.29/warc/d2808511f4412349cf86c2e35217d9eb345a01b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583795042.29/warc/d7d9eccf8383d89f4aae39950e3cb88b407673b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/02900c83df46ac3db1db85be568492636ca01484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/03ab5436c61982da5857cae5fa4f669904a4fbb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/7935ff4b3a15999e296d1f3d753b19933347b55d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/9e889f9299f3f87b1e4b5eff8d50332326949729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/c53a9eb39d8e46ef7e8c249da1f70e514f52c8a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/e3d4d84cfff4f56d92eca821e68779c327ac2483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583807724.75/warc/501ec110c44acb2e86d61796f7b2ac7f71e61190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583807724.75/warc/564646f8538438cdcd27dacf55bf08806fe73166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583807724.75/warc/5e8a7762452d7c230f6d0196c0ff1234483617d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583807724.75/warc/c9ef251135d4d86d755cdf3a59b96cf4b575ce67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583807724.75/warc/f12b86de573e0865b29d6c7460d362c2bc1989b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/26f4c6c0a1ebac2d737dc53b37b867f6572049d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/658a1954b69f27c5829856d9f2d3c6a742d60380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/7efb49f4d0d8cb73b8a8d40c4c0ed3d3b5105843.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/9407d1d4d9a7999885f43ad76779c56652c6bac3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/deb6f195596f5311e3fe43da7ceb2f877222463a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/f88a54cafeba18fe1b2282fdc8a3efbb50f6261b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/13aefde120f98af8fd4326b23a5ad4a1b43e0807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/4615f45158fd803cc4275e1d19cf2000f25ae9cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/4ced88db31e1f05d485a3b0aa7e69cc21f99f6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/656d3be51ecf3dd238491505ad1018f18246d81d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/97723eec5edc74917626a17ffb92d27e57e309d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/b1f1a008bff23795a55570ed1833dfb0ba5c1267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583823140.78/warc/01d2260c22137f457a98ce5ac9c3126b854bf96e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583823140.78/warc/7b6964fd0bd3c56979bc62e13c7e9a7304d3ed4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583823140.78/warc/84a79c85cb34fcf976131e27294d3f4351b1b80d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583823140.78/warc/b8a4bcafdbdc44d1b74cbac4abf77e211908075d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583823140.78/warc/eedc288d9b0d3627eb0924daa3ed0c45a49d28a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/11eedf45852b489a86fe4eb856e78f703ad303ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/3a89531f38e9cfaa7dc6ec1cfb8a0e0a4ce87b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/46521fba4128ea75bf93b4fd9e23428e0df183eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/71262c9e3b3d71ba750117f98fc7d164859739c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/a374ceba6a5792b7dbb0986faa3a0c18f7b47deb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/fd1c5e4d75899b28cd8d719b7b8192639b279bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583829665.84/warc/21b88b77517d58e4008ba34a4d3ad4a56b907b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583829665.84/warc/49625eb5ddbd1dce71777d024e7ead73720352ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583829665.84/warc/87a22242df2481c04b0f22c2cf9f5d14df1c1d35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583829665.84/warc/dd94f0073a78167e33f9d015121413bf37810554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583829665.84/warc/f34dd2b0bb6fa51a097680eb4e92dd280c204572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/080068fe37e9465ec6e0331d8b8e2b160c91f167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/2a624d97e59f472fa089b377d2c6bbaa1838fd7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/8e48fb9039cded1f42bd548033160b0dfe9161d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/acd61a1c371c8b098985298d5e95a65bd7d28557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/b785d4a589815136cf38ab0c3b42f0ca7e161596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/f052cfeed41b800539372540b17bb603fa039c26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/16731763bef6ab5948088dd2a435a28ca2fd502b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/23cf033382717e5239456f12b15f58c174c765fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/378706ecdcdb23751ab49fb6e7671a20f7fec3c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/92e789ac8e9302d36031427097add1219ae7d37f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/9a2962d12415c253d2cc183fa45711e11c42283f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/ef03f307208366890fde806b38d524e2e87743ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583850393.61/warc/0afa30e1e3af04e077d87c0d8f33471734cddb49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583850393.61/warc/0d79b35b34f802ba28f0184efee69327661370aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583850393.61/warc/2c0b8874e52febd8f1e20f730f8710e0f59d60c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583850393.61/warc/6e73178942d8c66e4a426a2aa909a92d8fee8fc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583850393.61/warc/ebb381b795d238804f73cecbfc438ce4033f7209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/77cdd4c46d3078c1ba686e5a4772ea35fc07eb08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/8aab5f2527c2246d0f68356a38eaf30cc7c20c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/a7b2de59aa27c412a8d9839fba7ef33ec40ad228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/beca6a710fe31481b588f47ddc3eed8f1731daed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/c1c930e115bbca417c4d5289e97d63a9682cc23b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/fef01ddace56c91009382934f6040702325a3ed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857993.67/warc/3ccfbf5fa116503075703cda0528faf9c49b4f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857993.67/warc/a755eef80aa2bbd83601ec021b3ac433e96f691e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857993.67/warc/b494e204f75f1c17e2de980a72d5a2fd974a0c5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857993.67/warc/dc632ee752bb6765be425ac63bc507aca59039c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857993.67/warc/e7d4dc58982905620162e00e41153c8938cdc12e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/0b4633bf2b1c0607777814d09bbfee700a69828f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/1e77fdfb290f389de82323d0ffd40d245630a26f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/32d9d66e3c2db24e2d425011bebae6b6a964bd70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/49c24720feec0d7745a475069afcc7208b5a6761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/e33f285948f6e8ea5686c7ca0421da62b1a25595.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/e96cd3810ce12b6d894acaef0d6b25ecf8849ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/4658c0e98bc31c967350c72dd363de9119f64541.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/6aeb3b94ca2f5f04c0a0e7ad0585d2a4d1493796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/7dca629a98d2c30dda0b557316d24d33361dda8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/be6fa1dd1fa9868b5be765617402370e4cdd8ee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/c50245f1e16686b63d1304793e2120517e22b1d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/d840d70c53f8028b1c8a2ebb0a37af5728d6606b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583875448.71/warc/3d9f788bb5b391026618ba414d6e9bddff873159.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583875448.71/warc/7c7a4235dc893d33aec18a19c6def6210f83c04c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583875448.71/warc/90f5a64dd884f747d7acab7149773c96bbc79c3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583875448.71/warc/93d43f63012b4df9bc643afb3ca1c28f5188c0c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583875448.71/warc/c66d9f122b23f3923a3942ec4f8f888de1c4a7a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/6eaf726d6525c5d81f430c7fcb7daed05ead449e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/8baf25fbc7ff6c7063f07ca2420d1bec5ca71f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/bd48eade4013144d1134c27530aab88f54220c9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/c2da6c03b6828aa4b367cf47ce38931c41969708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/ef44b4a0a45ee3483619a64a7ea920bdb35fa920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/f15981c139899798cfc93dddbc5b4c7cda3851c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583884996.76/warc/55a18cf23b0aeb834aeef9ce07813dec3699f92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583884996.76/warc/70fee007f457df707ea0e534266e5b8ba6205c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583884996.76/warc/93097d9f8c1707491503d07681f3fe3dd6298b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583884996.76/warc/acc3e2b460e3f65525addf9086433095e8d5a5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583884996.76/warc/d3b2b78ff24cecf4997c9e3f8c3d480d3881561d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/5464d008b03eb47f0678510dc4bc8c53aee03f7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/5bdbd1495908c8bd3ebf56a8f35d354223c1ac92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/72b342a1aa4ee7a2663404c4342baa6d031fadbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/cf94dd291b2b6980ff7a235a7738c51bbd4c867a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/e2e079749d0944b5104f0a104bc7b91817fa53d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/fc082bd7b8b0455d2f58bea8039827a67d2ab90b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/3737c5a3d0bb59c3443873cbb7eb8f6a60679112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/8e82a2e83070841e0913fe1ea0463eb9835ef089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/d17ba2302c5f9e66622b2243365bdcdc4bd2c19d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/d535a6c13c597cb35cac37d240c5bc8dc982478d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/d655d48f36b282e35d9122d8889fe676044c3bb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/ec6c78a37773e512faad5643d9d0caabf516a23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584328678.85/warc/08e644e32eec673bc910c9f61919587900121ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584328678.85/warc/77f4f00a5ab2af178033ae4854731151760a9e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584328678.85/warc/c150a93c5f0d5ce97790a7ae42c9d27eef46fa3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584328678.85/warc/cfa6ff8fa70ffbf57bfc4f2ef09b084c2fac46c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584328678.85/warc/f299d9f8d220195661f3fd1f1083cd185841cecd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/081f4cff98f368e01bb032c580dcd151b5749af7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/1d6393e6a41edd7944c30a90b6577a200a2308c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/3c149346e4d9ac3a76391af112f112d4a02bb489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/b7babbda7551a61da68e247a673076ef2d3250f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/c803a8942f86fdf81a6d72c3518d63398f0fe94b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/fd378b5661ea0f6aff43057e13310200a7610da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584332824.92/warc/1121428cdd9ed9f3bbd549ca2880e0b33152f38d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584332824.92/warc/181c831f60ace7b0d44407782bf8d30d02fb726b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584332824.92/warc/6c235f4d4d5536cb7f5779e3942f3bf16322c46a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584332824.92/warc/aa669ab2fde2909ef83aa924874b1a7d1d540e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584332824.92/warc/c448cd23530fc6a3760a18cf99f18a0913e29c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/212a419f4574aaeef442c4a0afa0aeb040ebe0d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/2443d7a5f6f65ff1556663b76cb7c466c3493018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/362fef2bbe3817fb137b3eabdbbf6b561a706f42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/a133c45dfc9f6002989b9cab34ed1389b614c99d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/f4413aaf79a0de527734f51eb24e14f34b89b661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/fa963f324376227134caddab41792991e2b22622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/3a6cba1874a12732b470770387c73e7cd6adc5cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/79af20b3102858f407d4225f8a1e322a4215e925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/b7fc8d296e3cab680043883a47161459d0b65c81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/dca10fd6ad3d290359f8da9f788b144993e09784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/efdd1c1c871f2e793adfb1dd012ba6c06399b7c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/f75ad152e9686cdb2b9176621042a0ab86715e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584350539.86/warc/17dab616dcb6cbae848db2ce17336d284c9e3a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584350539.86/warc/3c6ee8264af52f35906356841682f1775c07a690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584350539.86/warc/59715b076700ec2977fdbd547a5e4b59be23b3ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584350539.86/warc/a0af7da2907658fd673834eab57dd3218c4b6f2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584350539.86/warc/e8a3475f38c55723902bdae72c67998ff6abfb9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/18d30ab0bd825b8789f1227914f8c345413a5f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/4538cd4cf0fd2bf25a6e288c25cf472258dfc4d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/5fc8f5b4f3d77877ef8e0616dc7167d4035896e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/8d13ca6542cce0141160ba134af8ad49ae5cc398.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/e79a43da33c9fe7e7952242b130fb619fb8baf3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/f441cf1614433c84f287d0d5cec5dc7d2aeb2fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584431529.98/warc/0ce861ed699fc350a1c7c49d4228bbcb673d831d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584431529.98/warc/0ef9c446c2e069357bf3a2c219d79a4b12280ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584431529.98/warc/415bd87fae7333e526626e4eec688973766418bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584431529.98/warc/9bab10ef271c22df47801ed3922b7fe14913bace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584431529.98/warc/f7f1fec242badabbc28b3cb19835216c7b7d6968.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/4455ade61d8df0b663642dfcd9949532cb96232b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/56ec6f764d1e12f6d2520c2d450d580183d0cb8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/7c25667f028d6134a3cefb0d62a86f86ee1ae92c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/9c5e37e46a4adc9f22c80d89920d3defe71e669a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/f1862ca30bdce4b5eb1923feedccbee0b071f874.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/fce769feff5c847a0c2837bd2dba60c0bd94bbdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/400afdf6b198279b8d3c850531d8cf7bb310e0bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/4932bab3122f6c6b01effeaedf1b33bff9822dde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/57ed13ad5e8032662737c09b95126383f9a8d450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/9912016b8556bdd10ad539b0bb0062bc092d4b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/9cac2a6a9b971017c4a3f1aba16f672815d2b2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/c6949ceb4234d40e4f948d2fe8e7149a83ab623a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519382.88/warc/15605f21847848475e1f669ddd9ef486f8c2aea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519382.88/warc/4c0df3313efea5a78f5e911283728a96aae8bbf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519382.88/warc/90b52bbe4950bb30fc13f8665304c137e518e84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519382.88/warc/9cbed45a6ccf5b7816631b20118f5bfcae81048c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519382.88/warc/f80b9cf6e5eae273efbe137f5999aa92a72cd8ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/1f107a9e50dbb6a92c5933983b45c948b651e141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/65a783f7434d667d14cc2afcb9206c5f3af7b64d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/89a7cb8c0e5d4d5f83c584607b01541561907af1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/a1ca7726a4cef5636baa3cbaa1addb084e5e087f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/bad3ca3ebaa2441ef150da7e37e971276de78002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/dd85df4e24d49e6678bd77468e28ce0b72505b06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584520525.90/warc/4765b58443bcc5254f3480192e2599591a5c026a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584520525.90/warc/4ac95419cb86e2e93d018cbc1a101d717fae3c13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584520525.90/warc/81efd7410deccd185e105ab40e275e2ae8252ced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584520525.90/warc/c9120580ef48c7e8b880a53eae3e9cb59a86a7e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584520525.90/warc/e17fe148352c5a6bedbee1c49eb2bbba02efabaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/486f059625b9ce9a63441934745f4b0d16ec5e9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/5d02a3d1e2e9bb1f700e5d4e89a983684a6e9bd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/6df84fc8b936013610d39ae62e08511b50664e60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/befcab60e3c3ca36735c189df0862ce2cef0e11f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/db19053ba3b128571bff122ec1654d53ce8c3906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/e72d509088fc0f46bc2ee332b98eb90c51456ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1706aebb26d10dfff9af978371477105b97556af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/17e157459f90ccf2adc78457a02bc6989251a3f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/185e14694357faf78b1eb4be3dc78980543a23fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1e14aabc1710d1eb33e97ddb23ad2b4ee35bd88f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/20f59fcfd49cb9ae905a9f7e58714295093615e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/247899b5a8f94eb49355d1bdebdf1559bfb794cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/29c3ee733dcee92792390724b7323a43a18aa8b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/2a2f646ae88f59ccfd82fd48d412a926b03a98ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/31c36d0675a73aadd7986d220d232c2912d6ee34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/367481c53bb0db2dd1ecccf72bf127570c99e734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/368720458bf4d1778d973e3c4098834fccebfde1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/38522159c09d55fb17bb764cf5f0529bed6a4a1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/392017923bc94f0ecbd7dcd29010373cf567b5e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/3e1f911ec3baced06bb5dc4ea55d9cc07f205290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/3f49c3368201720a782c40dac084cacd13b490fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/401ac628559adbfaa41cf7d20670d90d63c9f650.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/43c35bca0ee165a617af78ad0544fc97fbb19322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/4572d7dbdf28c804e97470e2b31eaaed25e765ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/45a7e1926b683f569f4d4d3c35199633602d3e47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/4db7fb0129224a7ada03d3d21f46c30170799a61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/580bf18de339c333b39c29d7e315dfe032f640a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/59bafa80a0ce95176b0b587531b430be8cc795ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/5d63f067f75c1738fe9a32a3bf4f1730d9d30c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/609b24a6635e5203c15bfd3fa22146dbff48dc31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/615ed56104b54b525f58231571cfb7334c100b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/61ab07939dfe4d824b62de5af5994e06fb39c80b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/63db921a938563334d91a83332378e5b248e95a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/67455659bc2c1fa51c77da9088cee9d24b72f2a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/6883cd8f9d747d806a3757b467175572099b4433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/69984c66ec3ff8f31f80d553f711af10baf97d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/6cb26293efe5a9feb4792c3ea7ffa34b13a5d260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/6d65d3d18f1928f441c69d112659c2eebfc84430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/7075af67ebd44564c205ce0813f8c9ac8d236539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/713ff7071c081c054cc8f82cf2b66ddf6b20ba25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/723f2fc9725600dc073d86999d655af4dc34a3f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/75856b35195b5d6ebf167fce420118552353c899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/7b1179b50c8ec49c6f00112fb7715e13defbcd45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/7c4debafbe213b9e21b8b0c27c97ed71a76894b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/7c87214cfb6523775baa66564bd7072e2c2c952e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/821930d274a59d12182e884e0d54590dcc2fdb94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/8c8b926d7d638cd306f1a468f1e0bd21be596ebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/8d4d2350914312921253e4e1446b2ed97a385819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/9501959cbd216eaaa22cff77d5088e0385146c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/96a66268a795698cd4c38292a73e97efdbe25433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/9bb22371fc3b500f10c8641b3cf3217f7f4d0234.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/9fd18a40ded0c97aa32d7625291ab08d5b3ea153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/a2b38dc8b62f44a8fee1a7336f5ba990f389925e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/aa717dc854c35c25f6ef483bb4cf58c7b8e7fc7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/ab99f410d70848b7781bea9da636347d3e97ace1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/b1080d9fd1a7352333f3614813c1608cdd811ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/b2e9ad8ba59015026b553047f2e4ecc0556d6392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/b3c5f2af53b3b254ad0e5c9c65b6ee3b14378318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/b46620199353f1cd03ff9ae42a3e87884ac1a4d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/b7d64d6baf6a7b168c624b9835c3600934f22f55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/bf8b1fd2fe8d228bcaebfad43779d233f082275a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/bf9037a4f2275fc0fd55a8d1561449048949f3f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/c1ad76671127b4219624a381a4b75ecf1b6323e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/c97d8d99ddbe6901681db709ee228f8de3f4c0d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/ce9d93d05c05d4f296ab933233eaaafe1923c206.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/d683a5665299a19030942336f07fdc3d2da247de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/df6acc08b79b6b6eaa9357eaf846649a85b36b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/e2110311d4bea6cf27f1c05b43f37151d50928e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/e73f1b0364c773f5f73ab1c81d1983b1ceb7f9df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/e97b4d6b975021b033613f61bcbd599cf266b709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/e9fb49189b2b5e3567ea2a17b6c64619f9b1e393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/ea17f874466d5f7d59a9f023e51f7b02e90ab737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/ee01ccc72d25cda95cf664ff44b9f8dcbdde6fd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/ee49b2808c37c044d74cc97cc0fef1adaecf523e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f07b652ebc22e991783730817bfe409af5043cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f43c328577bcc46ca8d53f693629f6335cb28ce7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f456b97b69dcd51e5d4485f21e75aebc29cf7431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f6af3c85e51c460e9f2e47eb6b3795b2e1be1908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f6d9f2699ac91edfcc0d45b0e1d8d1491c593f64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f86ee0cff33f04386968772ece2709d42078fb79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/fe9c3e8d426e4c54bd2a812d1a82c9ddc7630740.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2019-04/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2019-04/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2019-04/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2019-09.yaml b/configs/cccc/dedupe-url/CC-MAIN-2019-09.yaml new file mode 100644 index 00000000..cf23e76b --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2019-09.yaml @@ -0,0 +1,661 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/08856f420d20866737b526b0f4bf9a3b097e07ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/09e99e0dad7d1577e17cf326e215d3df863500cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/0a6b473636a9a84d8a401802b61f17274a797b53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/0e7d14104cd4987f09beeb8d71db43a159474ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/0fa70bd98099e6406b636caf7fcf9ceb38971585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/109ebe7eb640db263c506806e4c84af4c9b87429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/5be97de6bc0f827524d7fbd5896d7d5abc65a341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/63d6a35a7cf612024623f0ea94f6ff0e40063713.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/737722f26877e3c5998bc772cb4f17ab48ed8a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/738e910d02170584febd893e863835f70ba8cd7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/7fd8b1a48775aa8c21ee83d9b1a33f27eec24b77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/b4ef2a2b2c01947917fe1bfbc77605b7354caa0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479159.2/warc/466f7032b01690474636e8a2c643d72dda80d01f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479159.2/warc/81b54bbd8f0806f7f690e5034c0e73132f384113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479159.2/warc/a783418ac70ea01e965dc512160f544c6a2aaea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479159.2/warc/aea793538e669bc5a3ab2729a72fe78f7def49f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479159.2/warc/bee61575602ceac8e21ead48d3783f0e76a27a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/38658df5bc6203f3e1687130be31024efe1c8b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/6359c1423b74fd2ce6425fa6d84517a8eb12d3e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/67bd4d16bda386cd5dfc5de5bdb4f923c46c39d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/9ad3f3a2cbadf638487166efed3500c658ed3ca0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/a1d07691d7fa9423b3b62bf91cad4277808adf80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/c57d06757b2db52ef62226f9ea0744f21ae168c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479729.27/warc/1f006fe3303b7d016b9cd6c0ed65df3411a7fece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479729.27/warc/48613179265a45962d8dc58e1e13ba527b06a84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479729.27/warc/48d979598cd6059fa094527d6d2491c11aa8db85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479729.27/warc/6ecc411caeb93360831cf4c082b4c141b70d5f96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479729.27/warc/a443dd6edae6907deba276d4863cdc5eee64ae3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/0887895e837c9c23cb4f765b0204967622764a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/7119e75aa6f2c3ae0ec07943cfffd6f9cb59f0d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/82032b4cedac16b1023973dc80c403f05684af44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/d66bccf874bb9742bdeab7196e805391a9c03f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/f08c0268d226d67bb9853f24f49cf01bb13b2e52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/f3d9de10eeda0d7362e82352f085068a61f6128f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/0e990a42d58e6fa408c48556754e96ff44b29107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/41ecd2a3bbecaaaca4f8b863d4903f3d553e7f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/44db7412a77d318898d794030b17c03dccb71d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/9e3cab3afbb496d284fe26b827410e85143c60ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/dcccc8b2b6f71a1d6f0ea3706c854d2add42887d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/e90fbc7e0ff6ba3d954d7daf94d883da2f842731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479967.16/warc/1d7c549661b83f8933fe20f3fa2628e7d7a96971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479967.16/warc/2169bc36fb6087d526aafa63bba3d24512e6c3de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479967.16/warc/3a546688aba53c555783eb54a5a6cca6e8fe2226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479967.16/warc/3dd5d28bbc2282237d9fc0ec8383c4d97a1097f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479967.16/warc/8682a640acb652eac53afc15ae10964822d134b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/0b076053100b211a745f37c4ae4d933de4af9845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/4aed4788efb83e61261fac668663aab5edc12d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/5196503ab75f9c4873ac284f518a5dfe9047e6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/c7cbd302f54f90b36bced8ca659f73a3a61dbbc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/dff314866435776d39ca4da332ae63e8725315c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/f8b883887500467deaa5f8fe77f7e3a5bb4f47b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480272.15/warc/2fd4e0538562ccf339db7cafd509aa0d5ac33fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480272.15/warc/a387585032db0a7b813f7a2cfd7a4da8261b1149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480272.15/warc/b1bb4dc20a13da398f3a147b5d5e40b101fdd499.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480272.15/warc/b4abbf9f69344b094544f19adcad23165f0e0773.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480272.15/warc/b64637136238274808f2b619295f74b5b34fbb9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/4fca3cfcb7a8c995ff9af35c87e502a2a3d9f846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/502710ad218d6a9db73f71060d85e1c5bc1c60e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/68bda96569a6079665005e03ee296aa28d9aac55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/8ac7c71fbed23a2baf66d77efe3119fca966498a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/92ccfff20be48f61e0eb17a05dbda72121bc1346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/ae9e698c5917e51158fc462a26823054f5a4f2f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/2b9fd64752a0f91cdcf6554653c908912817e7cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/71e8d6ef452e445bf603ed61b4162a8e87a0a424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/7ef80973c57abc0609617ff48ec23b10fef3433f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/7fdaac35d7a19a9a8dfd01ea13534bc5c03548fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/9587c6641f836ac9e027a3f49e46cdb61148b4f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/de4c24defebb8685a4955f25232be55955201195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480905.29/warc/22f7d83089c9776cfdc2a6e54f6fdfcc056c9cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480905.29/warc/83e90a6edc0ff7798f671b9dbd7c42ebfe8b62db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480905.29/warc/9d71347781083d322bf342c9158127b60bb01bfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480905.29/warc/cb3c1b2b7e90ba65cacc8406649d90d5abcb026c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480905.29/warc/ffbe3802e5a7c835010b8c7939e1950d1d2e0127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/7345304f51c213d628234fc87571d9e54a6e433f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/a191cc641a4198049c48ce12a437845f57d02cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/a389ace9b75b34c4e749395f8a11d046fb7af8af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/c2461216a70e9e4379083cdfd18c27e3f9a364d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/c7a0eae37634125ddc66a872604635a8b92bea8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/d20c9c3c6085c39cb6f46379f5c950b338188481.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481122.31/warc/2affcc7a5b6e3d7006bc17f15130523eba3a79dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481122.31/warc/3d1d99ce961ebf9ba2b594f2d5d799aaf35e5df0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481122.31/warc/483e4935a71137d897df682f8ae430fbf50972a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481122.31/warc/56c0051433d22e1ef474d716bdc01ba4ef5a0799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481122.31/warc/ed43ce34db1f79ff5a798e45771de3da08ee6977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/3fd44bca75524ab6a4a9e8e3f742c55522d8d769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/5ce375b5d7b88a234be7365092b64541c0fcecbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/671c8f9f0a431e3d7554216d9d4cc6472f9f20ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/6b8211ca3740e1ae7fdcaa685352227ceda19fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/98fb731844080bb51ec102669e1e947e6e0b991c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/e150f6b6560fdc1bb6f74ed2142741596d540eff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/126f811326f9cc98e8ab8da550a25f6b4f59f337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/7c79aecec29de9f0ecd6791a84a57f0cd819b3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/7fc1a9ae3493cdc6bbd6acf1e579c80c91d5f79f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/e0112af9808ac49a63ef180d8fba3d179a5ba0e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/e303312073d4d418f36a564461262852f28bc74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/ffb02531a83ad4a4cd4fcfbf06f68d5b9d11aa0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481612.36/warc/29c5c3d02d13f99dcf3967b3c439498977b053bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481612.36/warc/32d925fd4b56aefaa3c8e405ee07f8e406c2f9ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481612.36/warc/3f8471f0c78899a613d28f18a2cb0f19ef6af8b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481612.36/warc/da074ea93d43ce148ce68f44fc7e147eb1963887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481612.36/warc/e76b94a04bda07d83c125261efec21768705cf96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/3ec7499ce8b5c3a4e845b46347579227543e9b44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/49a91f11e88f94a0df32f67954cfdb2e959a1952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/af1a235fcb520bffae1fc479a08ca5fcd75fcca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/d063d6850d8b3c685c5f9b1e1549fc81eebed106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/e79854dc9c8e533599a75af51b08cc7f9c4b2c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/fa3e07d7421c044c1850ac33f0576d187d257313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481766.50/warc/2fe7484ec19789f66dd6e95db46fa47815c644cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481766.50/warc/81d7133d7bea17cc7e484b61d27957800e46b9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481766.50/warc/85d8d1ceb0422be77ddb2ba919243ce59d7925c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481766.50/warc/d0a5f94ea5eb7dd06cd39414fb48b46d59689e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481766.50/warc/f460a93987e75386f3083d1cc7ab52402f8f8ded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/299771c262a91e029d361b91b2c1bba65d13024f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/4daab928702ed51fbe2231505342174cf61eeada.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/825efd8247c5ee89019cf3ce33d78b65a00abfd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/b76423e28ea49bc38c9cec58744f83abc8baa675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/d994847c8a2466d12aec9fbf589ac737e10571e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/dd27487b25202fc00655f44bd3ffeb474a1631a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/05341d3eadafbf0b4953fd716530b646bd239fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/0989a48ace8fcba7f6b9d78c35f03971e62043f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/1fb590a5c444ba52066dde1241ac2f96b138a225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/5ef2e776ce6d831f3daefbf500d72268f31d24eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/8f26a4abd5f6ba66862dd3da1de1b48815419618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/903c72ec8461e156838b6e2f0b1537d937685a91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481994.42/warc/17f887aa5c7169804dd7491fb3dcfb3168918fce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481994.42/warc/4d9be9801de2e836bb11141c3bf242404dd382b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481994.42/warc/9773fbdf2c26d43e368624fea32c09b477486198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481994.42/warc/cb204b5d0cc820cbf38d85cdb170acea3d60cc4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481994.42/warc/d0a9d9808dde92a6422ade7101e34eed75002333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/41b5f059ff4d9de7a6cd611e9b31a9d053d7d27a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/900e2077473f158b442c47505b6882dc2c2107ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/9663332e19bfa4b999f3381ee6a02b71f96a3483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/c10747a42f2a333bbddef84a3b429d99d2551adb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/c9d577d02677a750c9e2443cc79e08abb0780ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/dbc642b6b3996ed109c9f6fb57d52bfd1e4a25f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482347.44/warc/3cecbc9466569318648071c5b50e3d17814cfe2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482347.44/warc/5120702d04d260a390fdf93b7efb6c2e6afa230d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482347.44/warc/b792e8cb2191c1342aa89d8193b5abbabec3908f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482347.44/warc/bbb3c240a2d395b5bb920dd83b95dda58557998c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482347.44/warc/f653f920145dccf0d892592cd0b90dfe7cd522f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/3bef386a22a4e00940e65dd37e4709d5b1640742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/46046e7a9c531f3036fe6c742b8f611fac05d5c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/6a70d2ed3110d935805180aa5203e05d3c4aa529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/6d2c16d592a91aad7bd54c9b373b18caee5aff9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/c9980b717307efa341de884f06e68eb6ab26fa97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/f66340ff6e0b95d0755b92611bf3fe01be75e622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/02cf83ec82fa18a07d759732e0be26c14db6200a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/05ebe7d153782a3ba8433dd28d49cda2c5fa55f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/0ca13989a57ace7d3ee5fce868506986171e4ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/20a09ecf8e865cee8bd07edee2c94ab28573dd9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/a3323940463c6983ac82959a2b63912c508a484c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/fa8fcb5f8ed24280a5d2e9393a6dfc07ae943e53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247483873.51/warc/3a9939198ff506246723c42bd7e9b89fb855e607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247483873.51/warc/6559adeffa13d856f50b5ccfdeedf2f3cb7d1563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247483873.51/warc/92e6ff7927a98565dcffe721e6a563d4317d9a91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247483873.51/warc/d698005818945dbdb8ea3b73cb96cac9f2848865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247483873.51/warc/ec687b51b1a27a1f55ee1a2ebbe5d4757a4503ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/3b51108fa005b285fb2170cd0d835da019106817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/3b51f093322969b42b97b944e572556ed610c252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/5ffe061d0f27f99917301628b0ecf9716e460ffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/7b9f18fb912b404ef76274d4168df7c439908f19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/89ed15fb12e307d3178f96992bbd7dcc4889a5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/bc109553330c4afbfec6a8cfcef1d041fe86a806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484648.28/warc/6f178522853e9c7eb370015be532be169e35c911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484648.28/warc/7c6385efaeb93a15da70178faa3b314f4ed7cae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484648.28/warc/a033f4f897fb88efd843396eacaafbffb868a1c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484648.28/warc/cacace9e8580b137eafd1d406faf8248b05e0c33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484648.28/warc/cbca28bc7c724c30e11f238ca4a0d3e6b9441404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/26887df57be91a991307b7c8c0b5c90c61f149b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/4defc9cae5707043bafd8da5156197e219a2894c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/57b58f89ae3e30a135d2a5b0acfb01a05687db9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/86badae20b7c470d5c1230766f1b35c274761699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/be4eea8925212217d1e3d41ba039c4ce57e00fa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/f376f1d9112019c341ebdba4c384b06366ad8e68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/6fef90ec9154e6324fcd67c2158c49d421516f3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/9d88d70847ac8d7edcf9a457f7d440511e2c8315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/a5f957b34195029cae6207ef9b427d61aab25364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/a97594c42ed7bb5af928189481dfb6f9bc238ebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/b1fdbfb22e8592ed765316ac5e6ab15503153364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/f287cd87da67cd291cde83f4fbbc6f9c30efab9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484928.52/warc/0350d0e26ac83c0458f499f1fbbffdf5e7fc0e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484928.52/warc/5d65deb3a9c5cb3ce2170613af44ad4453af2155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484928.52/warc/bf178f9e8a89e307f3d9dbf4f7c74a7d7ee6dd47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484928.52/warc/d5a4d24ab742c8b59ec8665c7a45d6083000f1d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484928.52/warc/fca34567aa40bdacd8f05fd14b778362346bb751.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/0aba7403b0c2cc1762a52c2e47412665beba9793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/21af8d13f60c1bf92dad7f747e65282022d9f912.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/9c875a5d3d994e8466b062f390d3e58b71182e7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/c1095599751f5d940250d2726876ab11afc6c3c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/d8d0a63b66c0bb58c965518691619ee895a406e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/ea474d6dd516359f6dbb3a4248ccb7575f7df7fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486936.35/warc/42afa38d004fa5bad10d13b55850b31eae7d98fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486936.35/warc/6b85a70c1caedcb8723a919785039a1c27803a33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486936.35/warc/9e2db4905633c7256a412343153e30a6c648f16e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486936.35/warc/d4d5a61283014d2b9a756dc614cf07e1e5d00657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486936.35/warc/e279da9c8d5f52fed4ecb545d7dc51dd7a0f95bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/0ff37f46da55e05fe9d225c36cb7c93136030196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/5a54317c9ebb9d71d98af86ea569876934d8f0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/608a6adfb543eda17b024bb48fc8d3989b7cd291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/c40086c611d60f57fcf810638849262b0778672f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/c94450c3253411c7f667e94b9428ac3545350689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/c9cc340280502cb7558379937f11169ccdc368fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/3b220e2a2d554fd555d071061c5c6047fa622b4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/406be567873a8e9e9bbe8073359a3dfb730df2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/4f5d10c3c80ca503aca556cc9f96e1054ac0e6a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/921fd41056f0dcbb0ef65a306b279eabe8728a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/a58a87bd2a98926243e7d2b38db0ac070973aa07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/ac1eac123eb3427123529fb6726dc2321c2cf20f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488374.18/warc/44fe84cfb086d90f2c6b95cba8c0b3b3a6c676f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488374.18/warc/956d4b12df0e5e0b05dcdd311ce661cb3836e864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488374.18/warc/af8fb653b7eec4502f181f73725bffe61881fdff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488374.18/warc/b77668a169698fa5ed761e1b0b48a8fec631f396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488374.18/warc/be1d670ab263098e173e70a556d5c464eb45c8d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/0e75ec12b3f1857e07dd0656e2d5edcf8b4a0a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/53a6a0d33339888b29a71d49b189aeb9840f862e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/9eda8e2ec7d387e1cec51ab8d3ab4a8e191e5344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/a722f3b40253db84016b0703a79d24c096c32522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/b16d6651d512b34a7e40b98e182fd9758ddba557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/be7d85c286472cd2b3724f06f4975186e15a177d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489282.7/warc/51033ff77da26e7b173df2ed7d86e213344a4c64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489282.7/warc/6c0f483b76c52958484487b4c79d97a388bd33b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489282.7/warc/d3982f84fe3c28f6e830e77f55a852c8636386ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489282.7/warc/e6e2ab48fc009318c467adad5dd1497e03ffe5ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489282.7/warc/f50d6e90609625bf33a5da0bbf92cbdde2c3a656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/1b03194a3359bcdf60ccf74dd4b6728dde0bac6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/2498a36990e4540d15c185f2b6abf3a3821944dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/308240a27c422c5cd51ecf8ffd1281c94da0f6d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/339a61e38bb2c12246c9aba4dbfac8c08ddcac41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/3dd09c9e64ae3cbf906f2b986dbbac018cb55740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/647b73952f956c0096b2e1e1b3875756e0e630e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/0af0db2905d020a11058d0740b73538c784d6925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/0c020a4347c92967dbb148483fedb4980de001f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/2a8e7c692613fb0bd0040189e11a9e9822e8b191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/35e21bebfe65d9718d0217ce44ef92012505ad30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/87d3bff3ed7e3e868c54f9cc5a39e108d2a662bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/f8593884cc72c6aa7f0d289ddba381d1eb3d1aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489425.55/warc/2b1d2d0178856452f28c524dd366addabff48688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489425.55/warc/4afb21e430085cd46a2fa4c988332dd2da23de9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489425.55/warc/7c6c824a72710f37cfce1ad9ba6c37e200da6c9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489425.55/warc/8138cf6a5df0e86fc1f61f863994adb257e6e52a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489425.55/warc/cf22b7bf2cf4f22a899a32dd02687eb105bac673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/13db915da9372e2da8db1f89f9ca5be434f4ff08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/3930e016ffe50058b508056c0ef145f718852829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/4ed25c508bc2b7267684fe6992c71c654dad3662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/50bc92acebcbd128a83e6e8dc92bf48b97d23201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/6e4f52ad9a8cf8bce8cc4e3052d4cb0a31d02f1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/b5b77cd9b1200864e833027df9a0f7bc7790cfe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489933.47/warc/36f9a6a66b2d97bdadf6f994860e89165125fb3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489933.47/warc/494aaa6a5378778fa4944fbc41a84214a4c14e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489933.47/warc/807978f4444d504088dd707281a7200596098283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489933.47/warc/cb5f27c10ad77fbf2a411a14a0f8743ea33d313e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489933.47/warc/f74e75cb3494c07f868e54fa2c5a85ebc4b72f35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/17436521c3661334262345caecfebe95fc6bc20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/26252242457bfeb4fa7327485291d92666caa504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/48262daf9d11b16e88f2a85318d57c5ddbe683be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/a5dea7a05fad9eec87676746aa788a9eb8b9846f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/b991d938c8d9bcb3265084e760208235054f11a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/c4f1ee07be73fd11d11f4676cb83a2da48be2fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/3a70a0ca522e3ca03a4dcedffe302de0a74071a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/89b96e96abd8cef76ee3a41f8512493d0c558597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/9dc126e74e848f6837a8533b52f69fa48f252743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/b0eeb2303a141237a87d9c7be6f8c54673cbe00f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/fe645459ccc5781cbcf106ad74f0e05bfdff7841.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/ff0fb4ee5075b9a551bc9f471d57a7c42acd54d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490806.45/warc/1b67e65ba08705f71a28b3142ebc002399ff3ce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490806.45/warc/36bea5676d1b0c6e91e4774c9a6c9b3412326c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490806.45/warc/89f03f8cfec8c6628953a55d820b2ecb51e135e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490806.45/warc/b810cdb4f3f2af5a5927b1afe177e0de88224c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490806.45/warc/e77c0c25c4f0447bff1197942c4d7fc18cdc274d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/422600a39fd9ac10b6036004200bbe5e6766a03a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/47e2d890104cbfe3f09af51f4bb3413b9f99b5ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/6ac899eb0429abd5d2c75553845fcec9ddedc577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/8bfad09820345d9ff7d8f774f0c6a53503c33f97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/bab72018440ff9a5d657722aa74fae0fcd8b8c0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/e85a33d0dc7a80ba97b4a3b8b63eee9d833e3f67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247492825.22/warc/03583f2998b2f217848b86085c0ad194ea41134f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247492825.22/warc/1c072af74ab9a943a42d8238f9c3ca256688fb07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247492825.22/warc/2293aedbdf71e74154048f12ce737a3dc008bcd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247492825.22/warc/5f3bc10fb09858e0777b73eeab5455b25d4ae14f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247492825.22/warc/6b575bf26ca95d15c6a25c056b6eb82c661885d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/038bc50d14944853cdfd4067a698bd3abe5eac23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/0b301df2e98ee4f11960f7609540dc095b834def.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/18d921297454fee13175cd774eda43a8b2f72a66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/388104144f8d955c9ac8b67e7f93cbab7e4664a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/c22e6af46c12dd97359e53ce20d19c7c6a427e42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/f499f7f394e924953b7ceb0da3d543786fb20106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/1c031a3ba36f2fee62eed50655b021df731d4d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/291c4a0c77acea376a194eb72be06ae8b4e40853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/59ca1008a9d15ef9223c7de814fb0363f5be5498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/71d0f0050d0961070f840e033aee58aba1f70075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/e53fbcc202e6d209da70d68ed85ee1d12a8a9da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/e86193e2db3e3dbed08558f99e238a24ada065ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494424.70/warc/27d86006d8e9eac9dad6e37971b4b81ec8cdeffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494424.70/warc/400598ce804c0c1af616bfa188c31afca6ed9dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494424.70/warc/778cc85919c32c169aecacc954282f297b1ac167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494424.70/warc/90f80c00b6b9ea920a62bc87a74a0928d8472beb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494424.70/warc/ee67f7312fa0c466666a6b887c643b613d0be6b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/245efc9a4bd8c8bfa692a9032455bbec803dd9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/721575c432b7df04509bfc70052bb5e5c222255f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/8928e96011c83ec18e64e55b0087721b7e9f2410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/cfcedbab4c30b781deecd3e2581bc695ad41db34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/d38cb1a4911ddd88be461a4577054494428915fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/f4cd0955b9ac1c88abdce1339eec3333e5164bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494485.54/warc/6b925879f588dd3d0b9779835f33132ac7cc1298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494485.54/warc/7689807b54b920951ce9db949b3bd7449b8fa1ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494485.54/warc/7a9c2cf1b3a70beb634381821f1fa83b42a166c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494485.54/warc/7dfb317796795115dc63f1fc3df8b02a5393d3c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494485.54/warc/c42b499453ae86514fdf28f306b92df221e108c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/34b9fb92ff7bfbbc8687762ccacd9c84ec8e9e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/439db199321b993dc0f65f36596f375f3ff91ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/4f0f4a1c2feb2a46bba7917f53ab9bfdff690f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/b9dce0baa78fcd9ac6460a073faa61ef272157c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/e112631403e140cfa342c10db59ba3071994c0f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/f5d2831c9a6aaabbbdd9f0e7f525f041f09dce16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/2c00ef09aed09313583a8fb1ce36d2c81e6d7691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/31f0128711dc7f517962342ed5f91b7aa9a2d259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/3576875ed655085047b58de3005f1be47abfd2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/6b0227e943350a8199a4613c74396a537d2c0409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/723af445a66343d6077bb7d6fd53556da4ca947f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/c110d1ddd5617d4c139881a0e2be3a1ca134d25c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495001.53/warc/0cf2d689c49c64bb9a623d1586bed03659448c7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495001.53/warc/3c998f7175c554342c6e5c05e2a68b17b457c286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495001.53/warc/426d7868b14fe68339da1f2016622031cbc60d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495001.53/warc/9bf1c2a7c88944981c61d1988b243ca80431c89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495001.53/warc/a985e12af7670cfce0df80165c5c191a1a1640ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/0c33a2a624fd11b562d9f5ea28b3e7bb8617e065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/3f6951804279dd13f45d8d08ebab01228d1e9a36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/63808b948e7a728fa289e20466bc3cb52385582a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/7cc9b28fa6c08ab23958460e79bf0c55d023582f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/a5e378aa7017f310d9b6f54f08743975c4c7b1b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/b6dde847134b7cf1c475774c89dda039f8aa94cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495367.60/warc/36994326cb951d66f0c7624417f57d1809186d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495367.60/warc/4aa70946ee61f2ca58749791bb2cd53d1b942f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495367.60/warc/9937e724d6a117e15073eb4f5219ebd17cde27ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495367.60/warc/adbbef8decd60573f58d37b116faa3be9dedac1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495367.60/warc/d006822cbc39f43f3ef6ba4cbf4d1b34195c74ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/6874fb4f1546a052177483c3ca81b85f294315d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/7fcf66f84294d608f806d8d5bf1699c255b6e204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/a1d10cc1eee4688b7757d179aaac59059e514430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/a8199cc107e70d32b20b2b2b0d3e80feacb90d41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/c200d8d33497198d49a4c967ea2a65d1c67ecfa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/f166c57aac1d9128f59806ce2d1d169bcc381b50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/004643a6348e0bb8b4f7eae06f4e9ab9a7b27eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/064779375d5b3bbfa3c1e64a31b2ec68c5afb703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/83f85777f1c0a89ef2e8c03770e28049f2c75435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/8f693931b1ab1863ca24f0fb3ba5190cc1715056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/a6a12e259c5265455afb38997e6489ad50a95745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/b1b9470544cf0e28e676776f738cf061edeeabc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496855.63/warc/24c73de73065cd3565363e45105b65a1697245a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496855.63/warc/32a8d561dc79e544a4d2738d0ae354e11727cce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496855.63/warc/38ccb228f9c8380aca144209c089b8a4a4d35004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496855.63/warc/52c07f3df997ac00fa24a2e516199e7b654bd885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496855.63/warc/b0d2798d8a4e3330960c46e1c328a1d3cf051c78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/32e6eca6a701c1d1b9f1dba9c8a5f5db703e1d3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/658757e03879a4ce5baaa98d42154e344aba77f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/d060e016b909717e6e9ec36d20d84965bf71e6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/db9aaf5055ed943c177a9edc78fdc5b772a03a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/ebb9b6a2f9b97c1da7c2b5093f35dd48d3b1a203.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/ec1fabe02fc1fb97a02acde614a03f51486a66aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247499009.48/warc/24e38dff3e5a956ad8b4d3a6babd6501a3cc9b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247499009.48/warc/35fefd58cbd9628bf7e8e8eb3b15d74e6830ef67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247499009.48/warc/411c79282ff928190868c6ba9534a307709095be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247499009.48/warc/55620d9f881711b25ae3a370e676778bf1761482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247499009.48/warc/c4cc5388f5662dadfdce6f86fefce8f06d540bc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/054fac0eb65a21502a5d8bf2d6644e64496461b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/193edc65f5f67c7382cbbe723ef0880ce760b6ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/47e00032701fbd0847579e89e9b670b784b3c794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/6ebc0494171d476e548b2c31c78b248bf8ff74b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/7efb93c6c84b2f0fadae554a324ff58c5cfb808b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/9ac6c9fa7dc167ebfb5ca5541005e46634798a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/0fb23979236ae5e5faed5403ecd59ac608518440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/1a6814f00583768ac144dd263a95233f7c383447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/69671cd2f01edf4731608de912992a694a681358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/721a6d345cfc9bb370069e8c90ab08b4e094d88a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/9ca8ac5cc70bfd2630f8b31a9e18d9bfefdea33e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/d47afca3b8d1e1f8758b9a7ac98abb713ade72c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503844.68/warc/4b15b3d8309ab1ec32485004463a9e1dcd2bd147.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503844.68/warc/680324b413f0f4e612bfa0b88232927b6f483202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503844.68/warc/907e8641356259fba0c380d98857ad7d59e1f801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503844.68/warc/9bbc509357593bd543cd13940c764e58f495b9ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503844.68/warc/f16c51fcba316bda25b9e72e65f823d4d4a16afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/0cc184b4ccddf1a8d3b6d9520eb2dd575a1c3c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/38efe515513c4a83dc1b5a5a55fe192960a3aa91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/5e5f081fbd9d70c0a8659d7673ccb1bfcde848df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/73e16586337185b2f960d65ba5f1d9a951db0a3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/7c8725f64d4eefef981f9bf6f638f39b0ba8550f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/f70aa6bc661904c8b6d2ee45dc56328eaabe3479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504790.66/warc/00abf931b0513ddf83e2fc945fce6c55a6f8ad4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504790.66/warc/06a646d48f151b0c99eabb00c100ed13c58ce6d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504790.66/warc/23b052a9fd9e6210fee0632e78d5c801aba535c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504790.66/warc/ba4e6c575b312b30f71fc40e264207a9b5108546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504790.66/warc/f157b8a4314aece51effbc88152edf5f201f0ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/0186906da424061c876cc6b3913dc54b4d3a394e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/0587df4cffacb72f44b1b45c19f13fd5ce76ae9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/2bd0e6ce59680ca225558df825e2f4f99a15b638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/7687cf7c159ced507a248ec6524b6201eaf4e842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/8e78d383b93721d7593328cb913067233c5fc9ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/beb6023807b37e2abfa81ba6839f0ffd6ddfa1b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/02133efd596caee210c502d63045011613be0dc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/06a5942aa564ffb8af04e61f42ebbd292e0f3fe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/159b427070cf3b4c134beac9ec8250ace4098fbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/3cbf8e25b7d199669592ef2d0e3f211af9ae540d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/9f10afa20316e340307e532377c1c681ecf10489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/e50f43e4f9bf71c0543452ec23bf211dc06294a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247508363.74/warc/36dcd6351b628ea8a1e83983439c7f5ecca3cde5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247508363.74/warc/53eff403a9c874a496b119663f077f74b684aa97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247508363.74/warc/85cb2cae08cea16c2c6413bb05de840feaaeb605.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247508363.74/warc/87e206bc1c2f9a37f4d62d1cfabc27c72416552e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247508363.74/warc/8ee8c0703764cb234ebe437943a2da673cb26923.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/481d9c9cc4de76291c19ce4c1ee0137fc5166b92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/4d2efa67af5bcd8936cc28eb18d33a4d5a98b95d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/687eb9b8866844c564a94f4261103c014a2510df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/7cb5f91b633c6e0ca36bc5ced054ac67e4fa3b44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/e404731367af584f53984bc38c23e09795ec1031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/fceb4c9cca4f32b80052c106bd01310828346b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511573.67/warc/0561971994477f4b3397fb4ef5633178e6090983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511573.67/warc/3ce129a731cce83b45de8feb77285baa6f7197ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511573.67/warc/6296affad6ea46f15de7fb45fc061c17533ff629.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511573.67/warc/8a363112c7b51e4a664a9b340904ae1249e8df03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511573.67/warc/f44aa029557cdbb15918094eeb57c021ea359ee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/0ef7124fc179f1cc74d96d0533b94c5e29b70638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/1377ee93b3207f9c5e303f6f836340d63c4a7082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/4d87d1b7a2a0fe07b3f3af7ab81235a943323dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/8b1ac45d1ef489c0e9c9baafb0ecbf9345ec71ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/93e4bb19787aa9fb4b6e7e4178975c355fb50d50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/d90a4b8c8e32347eb7ef48a8c449dcf94d6b83aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/02c69549b41b9366c34fe008e8e9734728f28e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/0402daa545bf783475c7a553a6a6a2ca93e90f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/55f33d99c2206ac3b2a41980a55d581d0093f922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/96769968bcb803a24112453436b411a7d889cb4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/a38bd96cc3b70e5dfeafa4df89d71f827c6c443f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/b16dd61bf0ddd3ddbe9550a3118b89a87f810980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513661.77/warc/1db65a415dd21d782fa996b95d01a6fd951fa734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513661.77/warc/31bef2df4a93f2c6fbf63e86757b6de8c878965b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513661.77/warc/48aa83fd8b83f8e69ca5ea2b49ed89790f9d7a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513661.77/warc/7c8cafc69f1ba971b13dcff0732b05877c2fac81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513661.77/warc/d0872c46b9e1e74908bebb30a6da2ec8585ee728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/24ce34477416f224275c24780405fdf763e280dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/30e3c2a67a79ee67a2ea4861671ec3c1f6754ade.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/9fde88d53c10577cc751bc3d79fbe719e63d0dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/a0fc7f1f63c4707685ab096f4f8f9977e1ecae5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/a9bd9a6ac879db4e77658e986bcb7c187d10dff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/af22cc4fb2cad02f4289037e24e0ecb483740d92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247515149.92/warc/2f49d11993584e2536e6422f9acbaa6f96cdacd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247515149.92/warc/56597ffc0227bf2334e57a1dfb89a9aa74a119bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247515149.92/warc/70abe06e9c935a631b7c2852425bf5a36723dce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247515149.92/warc/798a8380b650fb1a0d0980da1334ba1e467615bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247515149.92/warc/f34c6dc6ec2fef410d1e191ad80baa1c5b1377a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/16fa11c6db576ae77c80c676474168301d4decbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/2369cba0d86628caa13871339df6c7d6ae9fe32a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/26dbcaeab5f19f0a9f7a27543502248dcaaa15f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/6095698b0980f23669b32fbbe502fe6789cd1dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/6339c7e2ecb0f30f5f493896f405669289c42bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/a672068e0f80362ffdefbab063893aaacf005356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/406692a4329081420415486a8e97bf3ed4155e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/42ba362876790cb86d2de093d30934476b7a3d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/abeef11b77b5f6e3d37fa2dd6dfdae979be54917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/bf5aa0a734cf967af46ba48432bc5bb450adbac2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/f69e44a616b24ae9f8ebf6d132bf4e45986b00e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/fe2d9b549c9d437d210cf7458458e37da5d727c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518497.90/warc/4f2a378a47e336ddc29cba551cc31fba8245f7bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518497.90/warc/5ac3fa358b4af8b1b9b699d5775e88ef475199c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518497.90/warc/b28ed95c74a171cee0bd56d53084dd8f350d9936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518497.90/warc/ba337ca9d6242b6638980e3d0a9e1c9773821ed6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518497.90/warc/d1d0a15918184ecd852d8ca2fc3207e1acbc4ac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/0eae0745ecc4a50032d4334dc458fa5610785d14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/5c54f6989d426b08ceb4b3a9227da3773083ab2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/bb22dca858f18c89b32a0628493b2ac34e35c2cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/c7dc9c3bb4b809dca00fd69f37816ec91859a27c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/cfeda18926e2a2a5efcffa6524d9fb00a0c5ccdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/e45cb2674abf47633f8a86beb1163289771a574f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247526282.78/warc/2a56ea80b139dc48019db1183c452c24dfbe6664.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247526282.78/warc/a0f911fc024016ed6453e615e3a30dc1971e5583.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247526282.78/warc/bec90e3edf18f4d48e879261d9ddb9bbd2bb900a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247526282.78/warc/e70ba1a2231ee3fef3377ac47d4cd5d187040e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247526282.78/warc/f12d450ee764f1bdd0016c974b3801040850726d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/10da2077e088239c73a8fdaef4ad1ad3d65d2b2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/5e57ee8fb78f4379f56ed8cb1768fb7013bf75c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/6f1a0449a0c56b62bbc7db344a308668656dcc0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/82de7474e292af35e94131e3acadeddf01062cd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/831ce1ce833e26cab78aa3caef9e9f411beedfd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/98c48c957451d5aff82108865f86429097a9e2b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/5ac1d55cd0abcdf1c23bc656ed6cd9aa2357c51e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/79fdd72960cf7aab9513133ab23a93efa3693277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/a13954fa2cc2c4c72efaf8df2643e86d8b7e32b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/a1848cc378aa8c02c5c179d9a589556da8975113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/deeceb739399c6864eae2ec1ca0a4fba3dcd5d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/f837f9c8b00edda52a8c75b5ef4999cbbb692f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249434065.81/warc/036b7c5d881918759942d01eee2844a0199f059f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249434065.81/warc/6be1ee2b9d51ffade1ade1fb0235cacccc89cbce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249434065.81/warc/99ae06e27d2bfd3c78cc09240564f712107abd8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249434065.81/warc/b30366c35d3dcc60a788eb9bcc02cf6a87c31420.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249434065.81/warc/b4a31fd3d67d68b6a531233eff8fedf76f6086cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/386fc3329e0083246741bdd0230bed2a441ae055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/3b83814b25730ba94e26432501345436317765f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/4523ef490a28b94c306c76845716a8e3c47866d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/68102ee6cc23c3b4b085fcf4b3870fb6f80db3e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/6bb4b4841620b0130e4a99ed13de14fc23779456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/cc99b124e7cc4beb49075857d1e805604e73a5bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249490870.89/warc/198089e525e832ca4cec323aaa092aa00590d73f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249490870.89/warc/1b3c0de9ad7a8f1e72e3e9e3b6224cdd01e6e041.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249490870.89/warc/1cac41e3a0870abd228819629dddb863054fefd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249490870.89/warc/289e7abaea952419937fd0342a19858021a4a118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249490870.89/warc/fb694101c22e8ac264d0f59ff15b5321055e2dc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/667842e19e823e92dfe3c3ae03a1d801ca383175.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/8820bece9ad0d48f06ca455930c7d8b4ecb1eb8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/a213c7be37067176302ae4b83d14a4ddb53ae690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/b6d0738325a70697e1b5a4a021ef203bfc1657cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/efb38fab1c1c3a905ded6ce1bc28ee909af19c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/fd847b84b36be605e76d8dca80a8d8b4f49acbf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/08014ea2fd58002ef8a03c74edc24f9836776bb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/54c80548265b70beebc5e0bad2167081508a065c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/6c1b24043b9142494c2bda9521b79399a89f5d0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/bc06ed700af2118f93cb8b01074642fac78d05b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/c4081d14865c7f39edc842401536a35a9a75f802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/c8baec598cc938e1a7a9d5e48f923d28d4af7d26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249501174.94/warc/03bec7ad860472791c2e5a7ea194c7c80ab8f95b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249501174.94/warc/1163d9f88a8d51b227a307a91cadc011b69fac6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249501174.94/warc/896f45b0f736c2ad53fa22b5a3d9dcd7131e194d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249501174.94/warc/a3edee2caf9c785affce32bc13fa585b59538bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249501174.94/warc/b7082fbfe232f1e5d7fbd3f91b54734d410fb89c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/166adf93b6a7df620a9cd220b522de191ad434cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/6a39f82c86b01b38ebc40338d06561fd7ecd37ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/80d42245d78049be67001c7866ad0d2347d2bc95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/9963f2186750c5863f5e88412aa18deff5a2308e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/b55bf0679ed905878e1358bbeab3920f0ab2f79a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/fa16ece6d83434e4c580d45873dc90818d26a048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249508792.98/warc/3016d87c1bc4376789213ab4a462bf9dfb16fa08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249508792.98/warc/50f230402d19dec25e68f8d509139a3fd2bc5b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249508792.98/warc/60ba1a960b5bfdcfb2c40f235276ab61458c663d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249508792.98/warc/8d940ded74cdd1290b42bf8c7120e4c7f8eec0bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249508792.98/warc/dd03a573f1c4e1191e21215e69cadbcb1495ccaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/024233263a3b79d43185ac68e99b625a245213ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/060758e782dc6ca92f0f3b053c062cdca556fd83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/ed5f7e6b646662ec457f8751cd3611d3496999fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/f527d355cfd7d60af846d157ffa7e1156f606774.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/f5be5074af9805e2072f5eac25f8ee263e72adb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/f784627fc55dfa3617f3dccc019d5b24c0f44c29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/14ac22d0c5f3462bbbd7b2ab160b1e01df06ec6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/5d94ede847ff6f194fa53842d2fc5c1965175073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/69e0bb8733ed322beaf008869db084b64e0d0ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/87d16df81a2a6ad46759cf9c6303d1a48b9cb50c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/bf676fd30d480c5c580d3b219ba5f8c949a0010d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/cacf49088cb0d4bf9beca6005f105cf16b224279.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249556231.85/warc/469d34a86559d9ebe036ac1c0e63bf10abf4d679.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249556231.85/warc/751c45eee06fc4169b3d57e39ecbbfecd39ba5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249556231.85/warc/aab8892ce8010ac52a94c9376923bc201faaae9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249556231.85/warc/dd37e306e32be322b9371f6692f191e16d2410f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249556231.85/warc/e7dc6864d55d7aaf15d0097a0229240c51ada6ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/287cdbafd030c8d331e4281b5aea1cc5e5467451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/6daefb5b1216ceef192c4385d98f60d247d5b6e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/7c1527b17d170ab4b59e65955b91bccfa80624e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/931303fc7664e1f513ed1748ea225b0c1fd32d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/9d4a21e122d78b4b0b314cf955d92d67494be94f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/af121527d6cef77c8069512cf87178d073f7efbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249578748.86/warc/47bf8c0edeb837d805be37da5b4db54027ffe0f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249578748.86/warc/4c86e74bca3c7fd033d134ff826ec1d1e5b6ce5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249578748.86/warc/7b0985529fa8f22e30e3eec039fdc5ff423aec65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249578748.86/warc/9b687cce62ac1ed430bdc9fefc62d078f5730b0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249578748.86/warc/b1f68b6caaef3470e5dea9a60f6661c2adba9de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/13798bee6fb7f90279ba740146d7b5b647b0d1c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/469bd943c1c56c003933ef1ba1dec2e0480688ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/4a5ecdb6037a771f0ba0b6968caf245f7333c64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/6839c1a4ee9fa088baf28f1a45829b7bedc5ee7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/73d84d9fbffc890dae9ac7c2060d5896acd0d87c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/ebbbf5eb25ff0edaf3b13d7130dffc4b937f0930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/21e6c2cf976dd9585fd56e6a60c8a7f9b61bd5f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/243215deee9ecb1da4f6d016cfc37edb3d0541d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/24892b227adffe6e940c91e2bb1e07248f1df380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/25031316fce2bafec5051cde1437a7d3d46ea4e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/2545186f30d5cb2ad7aa63172343168baae4693a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/269e072941d97af8a6fe3cd5e966e4b8ade6e3b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/299dd2e9b14a4b64d2b905a428219b12a9fbe4c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/2cd5498f834bd2e229922c91ee81fb3ff633a47c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/2ff9e6ec816103be3208742cc96b74ead701e9ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/30f4b91b55a99676a040081c248d0646c054443b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/31c27c77a9a89473717534f2580346f4cf6f3316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/3578052c1c1314283acd4a79ee6f8f1658b39879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/3b189ea306f09a13e3221c41f755605f8faaf34a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/3c005359a2fc43bbddab15ef91d6a7c04e89677f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/3da2f41bd02411d338f54ca9f1d54fcc2ba8b33b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/3ee33600201c7befc5c052868697145567599a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/4a8adf1ec4ff71423b4fe646a79ead6773853ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/50b14bbe109ef4d2bf535d84c5830b1bb4b89aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/525442e76814d27507d30048800bf6a3e183510d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/52a36d4d01f8d194d3bd251c32d70cd38a404b7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/550a288e4d9efcf48102d3a5ff2ec05c9b213d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/57535f380e92f3b944d222f7ac221360859a48c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/59c694a938b4fbb7921d8c43bd63e346b65f0361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/5c153082ad44a9e8d8afd49c7a43d4a3ef83dc36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/71d97599c1b063a42d15542f42230ac620804a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/75ca437fec1831c5a329adf3df794d23fbef5708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/76f6d70a1df2ee40627f8553f902fa640721ea9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/7add3259a974c32a7b5a4d08b0810c95f84c4b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/7dbacae6b217993657bef2ebaef38bad9e0e82a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/85af77b93e1efbe3d770a8e36f2269007131c3e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/868c65d99dc37fde93685a779bf4cf9bfdb21434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/88d69d8287704900b667a6e6ac819fe10dab7de7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/8a00a33d4ed99976210d52727d4a27b9f2920743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/8adb8f87f6dcae28260d9d7e966316debabf8de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/8fca6b7312ca0121aab2ba4ccbdaff0588f83dc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/9396286d697b89e4b1d443114371616bc58a7882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/948afbf3f62f314d6c7406a41bb6bf43745bb24d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/9759b8c76eb722b8cc32f6e4fa4d303e8690bba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/986ed3460d117f6406a939887f5db65df48ac44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/98fc663c7001e35f8a5977a051d29274246db6cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/a00cd4808bd61e71aa0dc4e21c8e2210f90db519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/a38d62aa212083061d8fd514d4fd47b2f0c2fda8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/a7ee7d27a2bf641c2bf91d4adb015bbf0e6df83e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/aa8cada15394355027d85421e2323b4773548a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/abfd8859fdb2f011aaa2a0cb065d1800b7085624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/ac13772986a435ebe31ba52e775b65d9f30411df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/aea136306e6de5afa6cb1019c4bb5187b5111b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/b0a4047af8730e13c47d8a2162da942005b70d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/b3129b7ce5466270ab9d580340fffea5e2149f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/b31acd9a229adf323b9ac7cdc99b8db726c9773f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/ba1a680681fafb535d9368bd4f513a9d9bdeb572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/bdfd7879c2300031916a1958b8c8cdd6b36dc998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/bf210e7c12c74c2d35a44a11a0ac873109509738.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/c2dee3de62d824f9fe9af8edc1fc98ff008387db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/c8224418f8e436a2894a62a99cc5a119d09171b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/cd6278a9533d26fe56cebd468bef6b922724e640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/cd72a3800d38712253d305929d05b8e3b6a4c3b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d28126b0db625fad117027c8f9854c7ce75140b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d2ab3c6d03c3f76586c3a7d6a404012d5c58e340.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d44e343e619ead137e441c339d61ab127e50144d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d5364ca536d401d43dd42b5820ca694094c1785a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d6426dccc7b00f0398fd7f907cd2c58b164ddfc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d79fd0661e780d704ac300a684942fbb39f8a6eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/dc0853a976570a78889219a980e113779def6a42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/dc506161eacdec805f6a7d0ed4bfb3b26f72f6c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/e249f21469ea8a8c053b9722d8e3450732932611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/e2f0fcecd6e5d3d8f8731e481ed6bc508d43cd0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/e81720b5a69f23b628bef00c5e3e7c91c6d06302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/eb0269857278157948715cc4aa3c2f9364d7a670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/eef942b32e564c00df52902c154622accb5ccdb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/f43284021864b6a8058356edc8771996bd96e5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/f7e2879efc71257865f2d2f4e8a505613cc29e32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/fcfa8b529452576761f0ad0adfdda53c653e1df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/ffef37b2d5279856c95ce10a90586587d4ea190d.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2019-09/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2019-09/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2019-09/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2019-13.yaml b/configs/cccc/dedupe-url/CC-MAIN-2019-13.yaml new file mode 100644 index 00000000..6faf9d70 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2019-13.yaml @@ -0,0 +1,581 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/03eade6b92f8e8304ad2fb80ff514bc65e1a872f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/058e2a82174455cdde6cc3d79ee7c9b224bbb9f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/09f2ee4272d94bc396568b0ed6e081b8083cbc47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/111d2511108ee17eb2dc1c017c98e6b6826d0186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/11c2110828385923365fa37629b070c3d4e6a1f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/11c6344beaf23eb0e057898aaa7a0294e2a6342e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/14cf7f324ca0568f067cdbe7d38d7eddc7be8d0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/14dad08c739d0af5fb2eae190dc6799f71f9ffd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201329.40/warc/1452e0d3a3023bbc9560e30f1026f8a00ec4b3e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201329.40/warc/1582bf7db2c4b0b910daec2cff19888b183a3411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201329.40/warc/9abf66aeaddfc7662351a544a722277906630570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201329.40/warc/cfe590db56671b6e3d73355e17dbfb33393746c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201329.40/warc/d69ea8695c2295be50b5dafbda7db2c3038cd0f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201455.20/warc/352f7398f80766b59c709372de22d29c127dc9d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201455.20/warc/4afc3dc6bb81cafad8ff7ee5eee5113b0e50b6c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201455.20/warc/8b921d532668c36a137fee8e3ab28695f8438906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201455.20/warc/95fdce0ccfab6694955999ae81b2f62c3ed6f9d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201455.20/warc/980edbcfb7643d0fa80eb8a49cd3c8dc410a95ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201521.60/warc/1028cacbec87add321d031d9ec9e11bf14f9a264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201521.60/warc/a563baf73f14ee2f1f8c93acac7b64c4dfd049b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201521.60/warc/df51f2a8d9ecdc29f1a5a519cb064b1035a0cd34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201521.60/warc/fbaa155d7fbb87c524dd9780934d0ff1e49bc4af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201672.12/warc/4c760b95ca5b94bcb9d82dd1bdc5d0b75e3d716f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201672.12/warc/524f2d209a1508d9407ff524aa7d5304878ea483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201672.12/warc/ac2b99d2b3479b6234e3653e71e2a7b18275cbcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201672.12/warc/e999ae96e01e69f236188f93fc0a0232f0c90f4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201672.12/warc/f3e0c42f159bebf8a1ee589780eb0e1f001dc2b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201707.53/warc/047634ae9c578807b659fd27e1dfee92481a3347.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201707.53/warc/6201a4ea0cd3e91fd2b5e3001e931dc4ebc53967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201707.53/warc/7f53e4fee4fd0fde517be7d098860ec7ff3c0eb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201707.53/warc/c95fa05d3e09cff51cabe9f976c6273cddb475ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201707.53/warc/f933e3fcff79e15e27d394315809e3b6ebb28fa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201812.2/warc/27d76660ef5c3d14f85b09a59653cbb8ecc1bf15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201812.2/warc/2cf2dfc387fc69d6e43111fc2be1713099f0f04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201812.2/warc/7657e1daaead7f8af9c4c1b85e723375a3b37df0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201812.2/warc/c81db2d6b467a79222732ccdbbf5573d752f5f1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201812.2/warc/e6548053084a4470dd03bb5a7597d382b16fc158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201882.11/warc/4f1b744824092b8ea462eb5dba06e43a7e5bce5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201882.11/warc/711276bef716e32921b77fb897135c8513888e21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201882.11/warc/89754a866e92c506c99e6778c4df6a9b75463e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201882.11/warc/e21327a44c2afec6e1d20bc1ab47c4db8d3e886e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201882.11/warc/eae24ea973e4946d52b7c5677d1ade3f2e7fe8f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201885.28/warc/31c24d90bb5698faf1ed68ed187eb5afac8e5d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201885.28/warc/52c1cd9942250317dea33547d236f458e67d3358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201885.28/warc/cbf6973e30d0938f183b5f814be07e95e2d26ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201885.28/warc/fc4a644ebd5f3c3be0cff733d0d1ffb428dd3ff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201904.55/warc/074259e8beaa413a3d3f8cabecb8def18c91d1a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201904.55/warc/8adba02e156627895b00488a7cf6568f67322437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201904.55/warc/a992967f4458892bd4fc80f72da2d9295b4bcffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201904.55/warc/ab3d92d65e675dce61640655c9469a9904f765ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201904.55/warc/b24cd346511d06eb5c8efd07ccbbfc8be3979656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201922.85/warc/2af29593f52c20187b3ae02c82dfd762f4035fac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201922.85/warc/3dac60f7cfe74c18445b4bc084c2aebbe1b7ff21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201922.85/warc/68a57257c20801132d9e272d3113229d07ccbb4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201922.85/warc/96ccb08b7a03edcc8884db56ad45435e1f108fb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201922.85/warc/af056cd65919c786dd3653de7a833b3ffc290154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201953.19/warc/55e1283924f3fd1104aeafb79b0bc712b92dc418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201953.19/warc/5622ce6ba3020faa0b35ba27697db6f0017c2f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201953.19/warc/82e8f12462bb4ee25c33c76e507fd8172f5dd582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201953.19/warc/c98b82c45a9a10b7230524798e04e5e93853a440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201953.19/warc/ccd7d7e2bb728bd6f5f804ce6ab7ee8f5f121c2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201996.61/warc/1059cb265ff264e4faf10584cd80def3323ee7a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201996.61/warc/90ded46b9adfd7bf0bc050b036a6048554a1fdc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201996.61/warc/e32444f1c54059f4fecbf42b6758c79fc059f3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201996.61/warc/e7a9c770520387f2e43753a718342ed54d5a1d4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201996.61/warc/f6f574bdbaa644de96295cc1e6cffb9376863b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202003.56/warc/216fbbf8b99c7d4b28288cb0f0d8f4dcb4ceed8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202003.56/warc/25df1070db3b2bb8e02c9a1624c3b2be9fb0c618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202003.56/warc/3382d46a41cb7c031a5c9c65eca18559ea817b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202003.56/warc/bdf3f67f0cf739b117b67f91815da7de2aa15d88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202125.41/warc/0c3bb551b48a821de80e4f8947517a2e6e5ac712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202125.41/warc/14cf3bfc0ede3b0c57fb3977ce4eeddac62244af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202125.41/warc/440b4f3035d52435e8bb42129830ffb3ec770f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202125.41/warc/eaf05f652e51bec4ba5daf6b387b0ca90be00a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202125.41/warc/fc062b348245d2208ce731e9ebca12eb6a9a069c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202131.54/warc/223b316fd251ba38024469307039c223a12e7b6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202131.54/warc/88c3e372ae96ccb1afe21e97c9c42084069b38ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202131.54/warc/9e6dc8a990f41b6ce8ff93e67f0dab328146e951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202131.54/warc/c8baf5ca35f8aaddb5ba89e587730b2908945781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202131.54/warc/eb461ef2a460db323229ddc893f291a319b93b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202161.73/warc/37e3d780ea640c898be5290d12bb657424ae5977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202161.73/warc/58a8da96fefa55ea0025a1612b333f72ea9fadf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202161.73/warc/693d300243ae0ce1a8709a64b802492b69b31d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202161.73/warc/8fdfd7b0ac5d5245dbe8ebf96d18f899ce734be0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202161.73/warc/ae97ab180c89b3565a71954feb854b3033b6968a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202188.9/warc/09f3436fd4ad11bc9c1ec89af102abaad26ec7d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202188.9/warc/4aa42b87ac9b39143b4a3bcc5ddb093e5a938cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202188.9/warc/6ebc81c426dc54d86a82cfd8a5a6dd8eb0beefde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202188.9/warc/adfe12c1748d9e508397c9355f40209bdf658112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202188.9/warc/d15efa0a6dd79930aa9eb0c2b3d7b009ba921826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202199.51/warc/43a33a967f61eea36b45c623aebf91a5c0067588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202199.51/warc/4bf1f1f4fd1ae9dac5e691bada7d363092be8bb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202199.51/warc/8200dc014275cef99069c836cfc84b2ad4d97b14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202199.51/warc/f046ef16a105310c1c304e8a98403afdb21e7175.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202299.16/warc/6ce60ce74dcdf7f9eaea997c0502b3342216a6d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202299.16/warc/840be3afe67466d5a3fa48de0a8b520288790ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202299.16/warc/86c5455f0e336b2f31b445dc0bd94a531cdda4ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202299.16/warc/bafce44e62e2becd9ce99549591e832c4ac572ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202299.16/warc/c7fc386152e905b7b561e4f8bafd02d47836e938.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202303.66/warc/189a3da5fb2ee8765f60c1d59783f3209c2835a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202303.66/warc/6495ac78f43b6a0cc682b1bd2d5066d631f5567e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202303.66/warc/7bcdb72d698fae3733280f0294dd0c47d65dc21c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202303.66/warc/b65f8ee5304b69974348d736a81b6c2322294180.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202303.66/warc/f3ba4a079dd3adcc04513e3f08b3c2cb8792d0b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202324.5/warc/24a8831a439557ae8e20b53c3f79720f8d2f02bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202324.5/warc/34d7f76e4355a834a96e036bead14a94a6b98e8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202324.5/warc/95a007cf7dc632e4c7ea9d897724c383d05a1939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202324.5/warc/c37c9b16c002024d798c4fd08078024fc2a70c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202324.5/warc/df00b7cd549550cf0e04a7bef91c99bc2b2eff00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202326.46/warc/2e89243bf83ef476b711ae729b938154cb888d7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202326.46/warc/69f84f69834a5908cc1582f2a128a31e194cdc91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202326.46/warc/9c2c20b67ac160ccd81ca78b0d6fb31ae0e506fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202326.46/warc/b88e0965e64a55a48a0e5bbc87a550dadd5a6d27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202326.46/warc/d881d576e347b1caf1e28e797eb1665b82137682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202347.13/warc/193d6089c4a1b10372087aa87708345ae25a8c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202347.13/warc/709241a5947326104df55adf483fa7aacf5dd397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202347.13/warc/911377ff518d52b6b36162dcbc00373217622456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202347.13/warc/c2a2b353a4c527804a2c1b343335098701ab3e61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202433.77/warc/22054bdcf77661d2c15dcd66fdefc089224d50eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202433.77/warc/74702dac79e2826e31362811654a5c4801f16b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202433.77/warc/a46c7dfbf0a8d8b62dccb7535613c5010d5c826d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202433.77/warc/a4883a8fec7f17c15590b5ffa6e5131ccb698c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202433.77/warc/c969865c9b2668533aa82e1985490b07832c3769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.64/warc/3b62914a8b302c04991ac2ae99ac8b481af315c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.64/warc/7354f9746ff77d3bdc750d3e76a39dee832082af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.64/warc/97039ff03c3f2de943f8cb72e39218f350e310ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.64/warc/9f450b5c20788a9043fa74521b453fe469147645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.64/warc/e1839ebc7c90242eab7b901c899eef6540157178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.86/warc/152dcdf480a399b2f053d4e17de0c012e6ac9fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.86/warc/3f74eaac39c454e4d424eb5df168961d16f82228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.86/warc/5b5b833794739963ad0735a9e9d1a0e34440ae2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.86/warc/a13644f55fb95634823d92812bb7142304208a9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.86/warc/a24e913256032fccaef282c3fd6797ef03f55ed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202471.4/warc/02433b7100d84a68c552ab07c82df3ef8bb80372.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202471.4/warc/5854b4f3f32a0f7d8090c25c117109c7c6a1b134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202471.4/warc/5f997916579f03e5621b46e522fc265b06b8e10b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202471.4/warc/8bea7e3475d8aea088dff4d0098c3fd1f8a5b787.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202471.4/warc/e6bbcb50f1c3e235d2b395232d7a763a3a7e6640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202474.26/warc/371b06e73abaae18b8bc2b770682f4bae943105a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202474.26/warc/3f926a559de2b4bcd3372793589153e0bd780dad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202474.26/warc/ab3993d044f6daaf0d6b59cc52d2ea72fbe2ef5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202474.26/warc/bc33a39f945d856b961aba4aa23a1f108e8f1d48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202476.48/warc/34fdfc6a4548906944fe2aaf12156151d8d166ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202476.48/warc/41b658e2a1517d098551701b35b3235e2051003b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202476.48/warc/6e541a6e2f144399e6af25499f2473b1a67b5202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202476.48/warc/c25f653e9b8b2e6c980490426cb8e7fd4f1e96c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202476.48/warc/cb3dd1007f24c25da7c6821efad6637e21a68209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202484.31/warc/68f473377c67bba1f1fc12697bc18ac794eea154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202484.31/warc/88f6b2effc4024445d7c1a83fbca50c47dff623a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202484.31/warc/a3a633891a50b5201eff1c5f8933274207729e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202484.31/warc/b1588fe2f55ebc719f82abdea43a6a5aa1794fa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202484.31/warc/f6eb0399479fa8f406a722117a63facb0c226e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202496.35/warc/150dae6abaa4d7032b6d9597cca4f8235657865c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202496.35/warc/4ffba58da6b4f0668d3ab9da9543e1e858be49e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202496.35/warc/deec89c5765e93130502a82f02668f58b1e5eb06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202496.35/warc/ec646dc76ad3103ca15ca53ea3cee49146413512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202496.35/warc/f551896524cccb7ef0eca40c9b61da03c2b89eb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202506.45/warc/64967b0ba7fa8804be2edd8837a6c84657a74b09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202506.45/warc/86e238173d4f096300e6539076ba373bf614d963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202506.45/warc/8762cfd826db3490fa46845f3a347bb0355af30d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202506.45/warc/87bdcf52a2ada7d8ce5d811b03ac346497190903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202506.45/warc/dadcb2aa9d068b8df92082a353a084d79c0b1db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202510.47/warc/001159b7fdd89a5090bd2d0e11171cbf6dd1e2bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202510.47/warc/0e87a561f781118242a9e38f0b790cbbd98c2e58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202510.47/warc/596cf6ef37c107b87b9260ded677c4b7a2ad2840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202510.47/warc/ea7b4e6e566dbe31d99355530f5100cc63f20d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202523.0/warc/0499702b07a30e8dfd488950ca2ee999267b103d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202523.0/warc/26e3e4666bfc050ed60268ff90a1599629a24a61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202523.0/warc/3f3aaa3cb6de419eb23891d44dd5591d1848b3d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202523.0/warc/cc2bfc821d4d6e84aaadb5e424ee4deee8625d65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202523.0/warc/fa200476e632ef6ca9dc60066a2d62d6dceb657b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202525.25/warc/14ee6e44f3ddce6a21280ce01a056262ceb7b26c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202525.25/warc/27159bf8b92eb7e3c3294edd94da58fcd63b94e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202525.25/warc/34144e18fd1b705815ac9db518ae97a60db47266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202525.25/warc/d1e2f5c2362577cce90002b7d73e2371b1b483e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202525.25/warc/d8724c5c0e28afaede2231802af6ac753610289d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202526.24/warc/045e16c12518d68426e1ce7520d3fb377a285b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202526.24/warc/0f809e0213115015a1a1b42f23825462fb42516b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202526.24/warc/d506200efbeed461351f08a2d27760025f4e4332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202526.24/warc/f37336a1328e4857fe675c7220bb2e95d23f3dcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202526.24/warc/f5e44bb9108553742c56ca1ac6104c2bc35c9647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202530.49/warc/053d60ecbd4ebd6dcffb5d53c3bfad3e6527f7b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202530.49/warc/51a400709b130a52e8def8b1c702874591a443a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202530.49/warc/6eec04bfb31a06f2f4d1af27939b885db6091603.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202530.49/warc/743cd9ef14f9fcf5e9f7921d37b5a944eb6d74c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202530.49/warc/fda0e37881ad0b74b08dd8d73eb7a622360616d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.29/warc/55ca4236da6a92fb0b07f286da66e052d79fa980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.29/warc/562c99978c3500b933313fd290f019bc36dc86b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.29/warc/5ef637622425e1474801da8bb3d0f3fd55ec6aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.29/warc/a7e1385c14ccb3a512027d22927db5186a740fc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.7/warc/3f33c0d465cd0c1f9a839dd2fcf9c6f59457e29b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.7/warc/6d5d7955d716af45c58bea644ee128c2db15b5bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.7/warc/8b1d2a2d372dcd3233ec1478944ffe7f795899e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.7/warc/bfa61d599739aaa33a6df789bb34cc3b47bd808a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.7/warc/dfba2127464cf6e63a8eedd279802ee4b52ad722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202588.97/warc/0e0fe5f97e2c7b2ddca925e4bdadc722378d1496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202588.97/warc/3a3410635c28a4eb7ebd50f99d6642669913dc0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202588.97/warc/504df7e65aa28e6801f52eb91987809936cc3318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202588.97/warc/ca13b900b1eac89e1b7662835a1e59d0d57fd362.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202588.97/warc/fbcc939b8170acd56077c68150338527c40a687c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202589.68/warc/035911fcf7402d4665b254146163d717100603ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202589.68/warc/33b7a138a38f166e7587bae54323f63e8bab0a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202589.68/warc/646067bbbca3ef0d4c59813f0d1a08f281bccfbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202589.68/warc/6916a3e1a3b2bd0e55e355cb3fb11150e07fd300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202589.68/warc/cc225205a4875558e1f097986d4cab55cfe8c3f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202628.42/warc/05027df4ff5ae07421714039b69310cd41e79eab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202628.42/warc/40d3775555e2bbf3b93016c8c83d7d024297052e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202628.42/warc/9b1bc634de168ffcb8bb67a6d76b5593bfbde70c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202628.42/warc/c164a192729509268a436750eca4425e98e0a759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202628.42/warc/c1f8fe78a7f75c5e7c003e9e5328fd2acf5bf9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202635.43/warc/4a37c58e55a1f6be55210fd88f5c96696fd748fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202635.43/warc/7f2f5c447129bce09a1cfaf503e67e77154ecf35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202635.43/warc/98a968611be95baee07b2eec1e8371d6f1614ae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202635.43/warc/cde32ee92d92f2a98dd75b84c48fd04c8cd0a79a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202640.37/warc/b1dd5aa8b06f27d18d6a8f40f6f3aea65495171e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202640.37/warc/be85a480d57d31e436fa0af8540f231c087ee867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202640.37/warc/c5d317ef363e2770b8a38722374a3b5d4d7c6c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202640.37/warc/e5b5013d71baf94a33ca93b126484a8355ce003d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202640.37/warc/e72b691a16672fd8f0451b0df72a890470e94079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202642.32/warc/11cbb6679e94483539fee4d51831b29c0da0f813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202642.32/warc/42dd4f85aa58397e5922adb210c0c9f6c33af744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202642.32/warc/76ed8522157f54811a0eca5931dbd1d7ee8d0508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202642.32/warc/8629628b9c853760f1c5678bb5f670bb42b7dff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202642.32/warc/dc66f4cf5cecb99d3a6657f5b6f5bee562a64acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202658.65/warc/0b89891977d4ceebad574250b1be6a8da33b86ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202658.65/warc/95d8641d66c26c645eb0dd52745b0d755f61e062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202658.65/warc/c956009593fcbdd387732990b50e2fbde3866008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202658.65/warc/ed8f1e89dbbc2a9f08ef41f90e3549d20e740d7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202658.65/warc/f4ebd0e5e4d2b6d12c84b525465862e3c3d3377b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202671.79/warc/24598caf4fe4e449c552f4ff4bf8520de9bdb95f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202671.79/warc/3536ea8fcee1c43ba05f4d2b81f2ee87e3c39930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202671.79/warc/58fe9d61176aba1d11b7a34c62ec5d32f94c50b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202671.79/warc/b8f8c4b5fad552fe3111374578f9f0e3b781a27f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202671.79/warc/f49ab5794acc7e3ebca94bddf449254aa8635f13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202672.57/warc/47c3449863823bff6b72f0a1a99bfc7bebf680aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202672.57/warc/784f029b832998fdf2799f29bcc39803e5667dd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202672.57/warc/e22c1f54769561c4de91328bdcb94d93d2bd3da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202672.57/warc/fc0a14ee5270890a94d5074ab35f0a5a89aac4cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202688.89/warc/20b689dfd16bc5b7f1ca653f3c36d63d4fcca6c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202688.89/warc/26ac5d034737ff90fb687ed02873f1ecd2ef4c3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202688.89/warc/51da28bf7ed0444b4528fb27129c52e6b43fae0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202688.89/warc/64ad093901d991154d57b0f7941ad3acb2069ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202688.89/warc/882bb6f4b7b0e93308b2ceec5b90c6af7ff4b64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202689.76/warc/3adf85ff5844e00cded75309301fef7a2a3a3a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202689.76/warc/bd484e2f82a334545ea7d278e45a412d0dd73e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202689.76/warc/c7f4ebfb4edf2dfe84d5a7f5b70f8f1970af02d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202689.76/warc/eb265833e3c50b9267ba8dc086e693a74a1004d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202689.76/warc/fcd4fe4989e57d3ae7891d70f08f656806016469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202698.22/warc/66801f644c55af93b4ef828cd9403a5276a37b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202698.22/warc/8648f3ed050d929b9641117935dfdc0e80012fc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202698.22/warc/a3b363058f68540d1ff344b3e684a17ff9282b37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202698.22/warc/e651a50efcc3469c4dcf21114ee6c9c95d4a340b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202698.22/warc/f3219c2a5e5eab61d47e70181acf603ed767378c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202704.58/warc/1bcea3e4d573eb1c3a9b248ff4020e137b441272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202704.58/warc/37bcca3616aa2e6b70fa1c596e45d81a961a16ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202704.58/warc/9c502734f0df23c7ef52eb4553f9d51637530e65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202704.58/warc/cb3510c17f41f285b8920079c7293036a467997f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202704.58/warc/d045bb2c1540c72ab91436f5f2ae476d18b3bb71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202711.3/warc/137b6952383e95ffef5645da0367a1d494d1b4b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202711.3/warc/14a0e3d771021990d7cbfed0b7a45f341573578f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202711.3/warc/8e5cf820f3ad2117a8d3d8027f07de7869899285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202711.3/warc/d96550ac7d2c694af0a7f6c4795667c889e2da41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202723.74/warc/3d0c774d5c230c21f71a8260224ef891e5494a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202723.74/warc/a00ca06392437051aef643c1d8408cbe1524df71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202723.74/warc/a34465e1e8ab1aad9b2b3d807d9019dd59b0bb1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202723.74/warc/b3ca04fcaf3f82806c17feb436865f3a648aaeba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202723.74/warc/bcc95699d47a620d125242f5f50efdea8302f24f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202728.21/warc/4370dcf7ced8c68d6fa92e9d0cef02b946326c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202728.21/warc/a123d6d2f87f040afa74ee88170c30ffd5a5fd8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202728.21/warc/be66716ca4c5841079ca838a036d785a6505ad69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202728.21/warc/c5129a89f5b6a3dae2950c055be13a31e0bdb9c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202728.21/warc/d7ac5e087d129dc147c7864044d17d24f852ba51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.33/warc/055f4835dd3650ae7f8a213a3f4887290ac4afe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.33/warc/2dd4521786b68c30baa0c36c3bdd6ac995fc02a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.33/warc/5263a28fa302d533ba399062c2513c6c559562b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.33/warc/723c33900f13369d311b39c02d6749668b7bd81b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.33/warc/917f63ad9788ff69e667fc83b2ffc6ef55520d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.83/warc/35368b4beacf1cf7f199baeb2ea3ab9ab1c128ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.83/warc/5da196d4039ddfedf190c5dd6a186b6f4fe8d44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.83/warc/5e19dad8d969228ecfeb2e04e2926ada83b48e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.83/warc/e0808af3ff79630580e88d31cb4a24b62c582b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.83/warc/e12d81add27ffbe08961e3efa6f565eb8f8ac453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202804.80/warc/1491eeef36725c857f86ae144742f07fa6a12c10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202804.80/warc/1f35bac9eb8c02a19af05b57295c43ef5d72e58b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202804.80/warc/6e3833dd619871bb454886ebb634fbe9b15027f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202804.80/warc/f9393fa86f4bba33982d07c493b840f70cef3dc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202872.8/warc/1640c86504cc76b50f070689d832221b843c6811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202872.8/warc/48284dd573cfc08a11385384d4a0f1f16365821c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202872.8/warc/b0dbfb7b9aa74cf11d855e725db1a5d9a159f7b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202872.8/warc/dc3982bf34a7f12da5e9309d71d36eb0d74e0a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202872.8/warc/e0b5a813642230e1b090c8d3ac3cc4ec377788d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202889.30/warc/0758a911d1e29a139122d8d6cc2096b1ff615934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202889.30/warc/558f06d338a219c883fd85c33eb32f2cf3d2828f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202889.30/warc/636c767440a6c7d0f1fc87dc335b29dc836e5ce1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202889.30/warc/dc87db176d212bca2f9baa0513a38fab77680c2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202889.30/warc/e65d954e354934370a8fc1bd5d7334be695d5e5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202924.93/warc/5efd3b959fd692190723659c06dfd223c08fdf0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202924.93/warc/ce8157792097ed2652b92c98113e560c874c5a5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202924.93/warc/d97a4d780330cd51b154a946206f0f9ce703d013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202924.93/warc/e0e2472a9cbd88a4c7c7f3a7922f12f704fcf677.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202924.93/warc/ec3926ac053d16ee7fa5750396d2dbe09c29e022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203021.14/warc/63169df8e82c54e32f4a7903e1e65e450b9fe643.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203021.14/warc/6e2689d19a8c6054bfab88f45da40daeef95b4e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203021.14/warc/8efbed64ce2b6b57c8b90835989e7b1decb55db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203021.14/warc/9ba9c13d8870b8cd7c06cdcfc10a37c42951f4d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203021.14/warc/9eaf4ac457ce98116ff4c0d2a8474c3703c7d7f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203093.63/warc/08c19742d53011a86529a5a9feb99fce21854ab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203093.63/warc/17329e43827d54528a987d702a5d91107c905872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203093.63/warc/7f97600160b691f3077b8c79b6da9559556920d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203093.63/warc/f1cdd41fd73e00969a78b36668b321bac04eb189.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203123.91/warc/671e149f399148e8ac4e6ab49280bd4331e6141d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203123.91/warc/67ca9a0fd7c342af793c194db1878d44dcc1ade2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203123.91/warc/70c79bd999956aa33482b8a4c993ac33033f2cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203123.91/warc/8706898162cb4283b10e0c826f89b811a75ceab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203123.91/warc/c8c1cedcd8fb639e0c5e1212ce229f9a68c52a10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203168.70/warc/5728023e403a9fe77877219d722c13460fb8285b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203168.70/warc/5cd538d2f192e31e596e2e2ae97a89b4536c7e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203168.70/warc/713f71a8645f14c7cdf461c2860f301b6e89763e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203168.70/warc/d5efafc1cb91ffb426e470080e359b60d241cb4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203168.70/warc/e6bd479e4d6905a6d56870fd708dd223e90f690b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203326.34/warc/1de7e2139c5f0338fdddb8c3013e62fa70ee4da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203326.34/warc/48c926c8760c66d0e62ea5f0d12780d0195c536a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203326.34/warc/5d7c2e793dd98dd8fc28ebe46a30572672ca4aac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203326.34/warc/ae354a7f8fbbda6973a658931b8a3d20af47f10c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203326.34/warc/f81e96457058f6df8f6243b3ac1c7486f24cb57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203378.92/warc/299ed00aa51ed08e9df522627c6421b6aff1390e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203378.92/warc/7a0202a982ff3914fd28f9c4f00217116347d9f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203378.92/warc/9cd698c083380a08ecd04a9844c306b79f66a55f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203378.92/warc/aa1bc5ce14756d3ca37fae4235ddd10e47201f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203378.92/warc/bb5cbe5b1241112bea3f0d38f9d110dc5e4fa334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203409.36/warc/9661be8b378eae66fa5d23ef8f56da7b47a7ecca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203409.36/warc/b3a560bbc7eda696f906703ee54cc5e85720c2a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203409.36/warc/d7d8b051b03918428ffa482162a610a2cd67fbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203409.36/warc/fa6f25635907d929af63adc26d8d3549eed02de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203438.69/warc/0a6ab1e4a1bec4e425868b811093ce2b870bfb30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203438.69/warc/1a531037773b521ce35226f7b372fddc063f2457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203438.69/warc/1c3e3eb493694faee0f09a7721520a9e45afae6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203438.69/warc/afef3449e9773d10d823fc42a5bd410016eedf42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203438.69/warc/fa68e777b8f85b267bfec533fbb60331a2b966d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203448.17/warc/3481b6c3658607d325465071ae939bcdeae71882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203448.17/warc/4dc17159c279ad1e0a9b92ca1e71d583b28d8596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203448.17/warc/7c05c41057e011f6825cdd9f23c07b851b5e6784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203448.17/warc/950fdc65a5b25fffdc2aa51f120b52e2a1045ec5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203448.17/warc/dc7da0dbc1f81653ada57e688a195118ee48e288.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203462.50/warc/3b752dbe280b1de85f56bc2e90ad0d2c0eb9a516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203462.50/warc/48d3e366267c20c8e3ad16271c85159563ea7113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203462.50/warc/af2013a7524398cbb4e9c931124df1af97d66fa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203462.50/warc/caf247c69245a1c3a4ad51fbe69b1517fc62df78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203462.50/warc/fefa3bf9bedd15665686442245a7e25bee2727e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203464.67/warc/0b39144a4de3240f4778dd30388e159276d498fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203464.67/warc/12662141ed09b78718c047ea471a8f752ff2e155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203464.67/warc/3fa21d4665566495c111bd9ce7e1aca6a3986337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203464.67/warc/40eb7d10c02f4cf276e94c4cb1e0a11e8a639f9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203464.67/warc/85af55347279b10f513ba99e7e95cc486f0a777e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203491.1/warc/2d313c471fe6a799d50f2453e93eb6ed770dc076.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203491.1/warc/3284eaaac42e2af09b5a55e23e63d73751739d26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203491.1/warc/7a28b0dbc4002f2b329bf35c9739a1ed9d54ee88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203491.1/warc/bb8506704404b5c9f4ffa86e56bf34616357f0ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203493.88/warc/415861a4b0923d5127802cce4c8266ba3871eba3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203493.88/warc/642a6d5a29593e450b4bebfd0dd3bf3bc47d9c2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203493.88/warc/b2a97e7d49d40faa0385f3bb13e12a0b20eb3f5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203493.88/warc/bc5bdf0ee563d3229f68e692b2c9621c16d3a574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203493.88/warc/d6c38d5a748a9ff93e75f38948e680dca98a327d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203529.38/warc/06b940a40d765dc9941d0d3e22c221451bbd53f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203529.38/warc/65abacc6dd921a55f5b63c16daf4f68ec272f1ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203529.38/warc/7e6c92cb1000c3133a974454c54526dfac760ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203529.38/warc/d5e7ecca8d53bad1cb4499925611882a2b8505a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203529.38/warc/e53a6aa6bec5e3934de9cafde943c0fdb9aadcd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203547.62/warc/322d0cd7eccdb3d599697610895bfe8555c7e706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203547.62/warc/4c84d85f743b5b350259e35a28ae82912b481274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203547.62/warc/be08351881b4f0c88303fa094b659cdd2000f565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203547.62/warc/bf681205b0a0652f8b0be9fd9308c608ceade800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203547.62/warc/e01bb2d321fb8b0a7a9b694c1df71ac6cfef7830.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203548.81/warc/26fa4eed75007910601ab387d0a43bd5891fa219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203548.81/warc/3303419e4bd5ef800d4a1d8aa8de3d662db5ee99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203548.81/warc/5b5273b7ee43b6e6d5fd8c2afe883dae69f8b83b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203548.81/warc/aefc9425b82a57c28bea9f6f90ad1e50410aa697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203548.81/warc/b3cd9a8b3cde865f75199dad278f7044e3c49dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203755.18/warc/27ec0708ad293ac51971646b3a8e03ce2cf1e41d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203755.18/warc/3cb4d1badeae82bdb09741dd8c62f01c7626af6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203755.18/warc/3d00856f0a3841945304508bbc0c992fa6af4fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203755.18/warc/8ef594d10723abf12ffa6fca3a0a22a9675d6758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203842.71/warc/2065736bdaabb05846ded55479655790251da2ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203842.71/warc/39cdbde4a86da5953a59c5e822e2f2412bb45a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203842.71/warc/67d99db83a46b36789a043432ea364a3307ba7e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203842.71/warc/8ff4cd258b42951bfeb464b3d50f8ecf28c06989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203842.71/warc/c749cebfc844348fe23a87c520d9d9d0d7e2aed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203865.15/warc/a834db30c7ea0409d4a2259c492b6951f9d70af2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203865.15/warc/b220a55909f05b7a13ccf3bbc594d33347f8d9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203865.15/warc/c343bbfc8ae3d3af447f99c6d7403a63ceb022e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203865.15/warc/d472fd852432b961dbeb04644fd07d2f5c470007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203865.15/warc/ff5d59b5a689549250b3ee88da33f4a442cec24b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203947.59/warc/1d92808efe3fe1ab5b08f428f7e748bea014b3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203947.59/warc/2c3ed767eea401c5390d238628d2039ecd402351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203947.59/warc/309030c6d98421092d65abebaf19d54487fa9a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203947.59/warc/ca733540947195407f624df7f0957e3d66274da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203947.59/warc/f82f4703a3769ccb0a0b32bcf276e724883c5e3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203991.44/warc/0f25962aa627c1d1ee6f20314ea70197201efcac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203991.44/warc/3eca642bfae3905e0bf439b2d3bac5bf482abb15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203991.44/warc/b9138c6cd5b0f985233899e22b5d9c587fbc7c90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203991.44/warc/c7bcad4fc74126f4ab040aea671f45d170cb7b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203991.44/warc/cfc30d7baba0cef316b567ade956d8014bda7c37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204077.10/warc/808eaa436f5762ef67a4bd92a609f975b4a58667.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204077.10/warc/cf5ded53b71e33af05eae55702b3ebeaf5c9dbc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204077.10/warc/dcf2905343dddf0dd991501da389da34d5198dc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204077.10/warc/edce75638a91d0c7a931d0ca261132aa30c2cae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204086.87/warc/1c9e699ea85a35ec662e025d5eae62b148ed2892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204086.87/warc/4764492c353c8ea7214d951f294b99ba58da5ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204086.87/warc/7dfa90c0e9776220cbc71f31be7dd680576e9e3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204086.87/warc/c6b8fe354b8aeee05e1bafdbaab8bd25fdaeff70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204086.87/warc/d63c5284b072f4497ee5263ed6924299cd3eb8c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204300.90/warc/4315c6e3f5d5275cfd43d1f8648ced92e3f6eb39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204300.90/warc/60897eb2aaea7af52480f37ea02ab5a3d4284435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204300.90/warc/67a2ca9eabc1db1f1c97dde81010b9d116159eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204300.90/warc/95f60e854cb4e77e32c95e4755f95a31015ab44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204300.90/warc/cb5cc9960fc360e11f0b23ce4672028df688df00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204461.23/warc/1a3bb97c3c139aa5e740c8b9cd26b5beb0ba9e66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204461.23/warc/89d5baca16a2fe155f6875414e267cd804045907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204461.23/warc/8a2eb97e9a7694407fb5700d20db31c59eca5f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204461.23/warc/a3ff422ab80138e25cdd7c058e24ff126572539f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204461.23/warc/fc948d13128f0a724df80fe7d66610d7f906a7cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204736.6/warc/67fb75aade9ff75d7c1985e52016fd7aba4cb1eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204736.6/warc/9d45db007b5d66783c2e63f4a0c9c734c72b92a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204736.6/warc/f4028bce442d012200e1d6683c2fd8e1ca05493d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204736.6/warc/f876142a6c116c2228133ad8829b5e1ae83010dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204736.6/warc/fe201a6d108d2b56238567463785209315017e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204768.52/warc/6250695e5a048a3ed746b32b5f9170302d77b087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204768.52/warc/652dc347458ca47538dadcd1df8b88184c83af03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204768.52/warc/85fbaf93c5fb44972f7fdc8818adc26e2663d952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204768.52/warc/abeabb3ba8b48ea9bab98c18a37e391061a307d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204790.78/warc/21f23515010802bea8752132f878f442beac4fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204790.78/warc/455d88abddabbdba1e605f11568651250af07b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204790.78/warc/850fc8a526749babd0ec871c44e23e21cb188812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204790.78/warc/89e6ea29377162cb2e56861441b4ba9d55e8a571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204790.78/warc/db31ad2fecc0e96afb88bbb6245c90e3fe2cfeb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204857.82/warc/4902eb1b96694f22ca4e502a6c4f0cbcc4de94ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204857.82/warc/521e519ad5147afcaa6815877efb26f371bc4730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204857.82/warc/53e32e2ad5f45ccbc8dbd7aa108027fc66fc0831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204857.82/warc/8279c0e7a594f4dd840c324e5cbf044f2a063620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204857.82/warc/93be91d2843aa9d7fbdd11c64643f25a4fa70a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204885.27/warc/510326a1490026231309a60d6e704c32d122881a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204885.27/warc/624602f4b6015ad25587bad5106de7b6a40415f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204885.27/warc/7e338068e127cc9b1baba1397689d77696acff55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204885.27/warc/b8161603c08728e5ede30fcec2390534f7be9227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204885.27/warc/c61d1b04048e5a1007376ac6b658358f73328d8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204969.39/warc/12450daefbde5b8466cc67d96fb164959bea634b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204969.39/warc/1fb0d61d2b265e33100ca8487c166a1b8073ca73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204969.39/warc/916ed6809df6027ff967a8839401a2335149f009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204969.39/warc/a662dddfad29e7139150431b5214d58e0cbb0aad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204969.39/warc/cc07bbe8acc183889a115d5106b701f1abef4bc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205163.72/warc/a72f2bd0aaafb306524684ffe786853dc209b957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205163.72/warc/b184fca2baad42ad18921e9b15aacb0437975c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205163.72/warc/f003331861f0e49cfa8511f410d17b8092282bcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205163.72/warc/f1a8d2dab988c419ddbe32dc3278f6ae02c59c67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205534.99/warc/5de53dec834443b23cffcf3e3c9dfe6e7778f3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205534.99/warc/61b28bd19490d2e5e874a7aa8f6ab11115aae05e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205534.99/warc/7adf94550dd8bb789e13df448e2e266192db181e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205534.99/warc/7d9a66a5a3b4b552ecdbdd041b67d5998099d062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205534.99/warc/a2c501ef5388b6445becbf9106be75a2c09833f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205597.84/warc/1dc0278244067803d6f10a2c4ea9dafe196e2a62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205597.84/warc/20cf421b32ee03052d70a36215de81b861c3935c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205597.84/warc/49440bb01a8bc30da80bc6c30f73c2c4340612c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205597.84/warc/7a2af33d1c72eabd1a2aedd7282d6e8d406c6032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205597.84/warc/9b87bfdcd1cd8f7f3bd87eade4d457a670f18e7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205600.75/warc/0bb6cbce5deaef0abf7202ee7098610ae0b0047b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205600.75/warc/10d28a47805bd30bfc235a2a90f47dfea4226bcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205600.75/warc/5a8e6d282aad58236bc2e356f9d1e6ac56e10542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205600.75/warc/6552d28908852f64b9d55a5853d4fe9593965680.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205600.75/warc/921802f43c8067020aca9fd2176f6212bf8f41eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206016.98/warc/0750d92babdb948fe719d3eaa4cc19d3fe90b5cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206016.98/warc/739f263655a2d3f920a0fdfdc37a0fc7ec5d3f78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206016.98/warc/811a20de47ce5a61b60c8fb0f58281fc88d7c389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206016.98/warc/82ac6870b124d7cdd5fb321346c9cf8a2d0a915f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206016.98/warc/db5c615c7abf7ebd114ea4b8cbbd0761f343b1e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206677.94/warc/12d38fbfc4b33d4414ad2746aa8073309af3bd9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206677.94/warc/ea9f2f3b93c67fceb3e154c660308ca836926cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206677.94/warc/fb4b7993f823938576fcfb7c59a01981ba65e02e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206677.94/warc/fea5a46d8892c9f8b87df8c7ce43bdb01a7b2109.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207146.96/warc/03506fe51b34b5a9ced88195794ff71d2b6ec719.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207146.96/warc/3f9be8ca37e6677fa88c3ce191d4392a6195e41f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207146.96/warc/4f8426e8a87238cc0c950d6df08dff7ade493305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207146.96/warc/a7382873c7a2be352669199861c379867522a73b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207146.96/warc/cc0deb04af00226de9f53559f7b4ff629ce10fb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207618.95/warc/100dbb0b415682956bc3a59d7eef0f016a80f609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207618.95/warc/5fba57b2f6858ca48ddf96d0ea49f34903b09d2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207618.95/warc/89edf3185e2012adad574c7a6ef36bc3a784a525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207618.95/warc/bb4755f381365c81deaf0f15d3edd87f8fad69e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207618.95/warc/e0c11bbe7bfa5ea51eb45218bc372f3979c5ed92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/17fbe3d600deb7d8eca2e7df3bed4fcc1cf7c6f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/188e2c02da9c5c1b92b34fa71885798616a14cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/19556b874f0df88e7b2b3edd02090713292883d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1b9269e70105a42d5bb5404ebf2f4405f7db1a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1c1a8326579cf3cc52b6ac96c44f032760d091de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1f777fccd70fdb322c463fe12dc3f5978d5dfdcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/2040f336305c3d07a816d6c0a897d65f35f93633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/264ffbf85fceeb0dc35d44433cf38dab66e9efb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/2736f82684d6c11fbd24a7b2b88f85e7c1bc46d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/29ed653b37d740490242ae16892ed775e9fc6888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/2cdd37e4c06a293873e5e420d972e55c662f8f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/3db8e6aa81636299f6f33baeea39b3865c6a149a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/422fbe391ad2b81cbc7e3833501d7e89b77063d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/435ac34567069b4e8c0a28f6013e17cd5d04c4fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/4548dc7091234baf8947b2523ef3cb5a6c3a4ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/46d4cd8adb21b830ebe910825ef90f9b265cf1be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/480b73347d9f28aa4c26435690d0cfe0198cd49d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/4a6e7cbb64af47f0cf5d9510fbe1f4921f65bee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/4b157b004cf297979d5fcb9085309d779278d0f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/4d5cf6d591cdce1ea3fdb496bfd1248c126ee387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/512dc838aec5ba49c5e62a8f06287f1c2709e9d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/545f8bf974c8b49a7b0752b4a91c5ae87a1e1d15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/5a2ccf8122000c7791b3ec24bd14230e115e0211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/5bf98a702b1b70f28a9eae0cbc51358cc308e44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/5ce0b40b2aed66a702822cb72c68c13682f6cccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/5f7c5e5655be56ed833fbd9fd4dc681eca7df55c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/60d1881c98b6007b336aac287e2d886e7af5e4c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/6337b08e259b5cd02b60bdd3fc2f3cbda53f91f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/6346588c56d7c026137a34f70b32a7dc3912dcf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/64d07de9147e998c50799a2801722824c15c4e68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/6d5f7d99a6bd95585dc3fcbcca0093e463bf4d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/750dc0c8f07ed9bcd119fa339137faf859cce5bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/78f16cd47f74fc8fbb639e71e40b842094e4502e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/7ac106efb04d614adb66ae2b52f022dd7a8b3034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/7ac1a6bc762bf2ee5ab440a5ccb98226982aa087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/7bdc0c4453fd040c696011c5dfd2e8635d27c5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/7edfec22d0fe0a3454ef1db5c2963cd7f16c6e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/84a7477e74885d0cbbdaf8d476f8fe850dc76539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/870a1da2d9ec73e52bf152d53595dcc27c4067c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/8b8b5719475aad1cbe9a3031b4916ebccc87526e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/95d0c7c9ffd88183d3aba19efabf5da58292ac9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/98fc6c6dfe64770e434409fad329b89cd07db47e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/9ef1e952cd97f9098b1eaf6a19c6c0b37ce9ffd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/a19974971784ad000ce627a6e32836c27074f2b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/a27a1b93a5f0ffb8a4086cc7d860f191ba92ca4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/a3570a53b387b77389425de9e6dbfc45c7878d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/a3674260aee1697247e2b323bd187a2db818e582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/a36783812e64ee76c67f617c8d84a17b931f0033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/aa52715a83add1af64848526937f3183bc14643c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b1397937b9f61408d834e0b5a306bc8ccf82157e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b2104b80457731cb0aeb72978de6a88dfc045cd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b4fb63c91783a9e589788c17f6ddc0573d4968ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b595ae2ffaf942b4eff327df642c0bb9b5c120a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b5a309b9c8d499ea49aa071d4d29101f5cc77940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b68be944034ce359a92e3730cae4c9ba2abbdc67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b739aa3fb7c9d9aaccd900e699b2fd296ec9baef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/ba449bd89547f4bc12fb5e55402dce4a3c494e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/bdb32c2097409e055284969bc624404c527f3dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/bfbdf3901ca7e8a6de09cf2cfb2ade688cf64775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/c612989721a37293e1de1d4c1a798a3bfaa972ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/ccd380c13897c4d8d5848fec960a10152dff4a85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/cdf987c1f03167d2d88a2daf08948b9f26aed03f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/cfa5e4876023576531007e422bc31d2083f4a077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/d012d793f235768d6accbfa0cedc29d7c75f42ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/d5daa68e10b814e708a66cd9277910ce6b7823bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/d6514ca7b1f6149fe26be499964d6c4bcac0eff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/d6e3e278c7ebeea8045a315c947277daf4ca672d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/d7a4d553727cfb2b20aed09643cbd3fc923fe577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/e749518c13aef1fd5a73435e5044d31a4defb065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/ec39fd8c898a28a90890b0b25e0359e864f9ae8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/ee6ac57155b296f8f146c22b4f9a197131011903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/eecf2c636669290e69f6c2495595e8d1071a7432.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2019-13/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2019-13/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2019-13/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2019-30.yaml b/configs/cccc/dedupe-url/CC-MAIN-2019-30.yaml new file mode 100644 index 00000000..02c427d4 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2019-30.yaml @@ -0,0 +1,581 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/01343ff42761fb71a6e89cd035e450a61927e368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/027e109de94118995d148bae2abc2625f030f1eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/03fb8b9cfc21f1027f17505cd7bd37a5c8438721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/09fbeef083a9c73c6709b3f343eec14e42495c14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0b13e924c1ffa2afc1a058c00f96f04c3a0d1360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0bd154107fc93146fc42f4923f0f0850f341bcff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0d01ddeb36f5399b7c8fa3a8f19c9357cb1971df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0d40f60cdbaf6442d76b4228f799c84a763b33c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0d7ef062dd9735fa9dec08f9021fcc06eb81fc03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0f7fd7a905f79e28ba7db49c93916f2a5c6df03f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/122b39e16eef4d24fd17c7de203dffee4391b149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1531cf6fc7c060022877331fea075d018345a2fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195523840.34/warc/18ec14348ab0674189b3fe8cbdab5a2998aebff7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195523840.34/warc/3e8c28278da33a5e7c0ee7df70210e72f483df25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195523840.34/warc/8cd2651b17c0795cd51897244b17b49de117abdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195523840.34/warc/d22fe5832e1bddd582893fb5aed83a3e553ece44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195523840.34/warc/d3eb1e60d43a388b65239d0b489b5d7ba970c29c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524111.50/warc/460d1a7e0921818508889b0f880c9bde288e81e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524111.50/warc/48a11d323ca93067485c6b00253ca5011d49f563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524111.50/warc/98406ec37074337232a3fa4260781cc47759f32f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524111.50/warc/a862927dd945dc7bb7c0356fe0e6c51cfa77eec5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524111.50/warc/c1dd9d03b8b7b0fc2eaac8b0f25635068ffd0849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524254.28/warc/36d41cf8db15e5876b10142334e8a0f2d80cd5dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524254.28/warc/41d9bcc7a960c9ae622ddc4f06b8f15c9299187a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524254.28/warc/693f4c588b005fba4304a89724f15112c5862757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524254.28/warc/85db862826d0d883718a7d9b35d58f38f476c463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524290.60/warc/804d7e2c94108f8a65dbad9ce0842a9a31057f8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524290.60/warc/9bc24a432a047a48e2fbc84ba524eb8cf34f4690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524290.60/warc/c513cd237c7d77694ecfc8cdd91bec1d4d006761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524290.60/warc/c69b2202bdfd54f40f6a4c3ea77fcb0325ae141c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524290.60/warc/d367af73512acfc1f1016961f580071f5631fb4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524475.48/warc/1d42c2ca790ab911e17f33cf5a0dae9734b9eb91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524475.48/warc/79171abd39c810fabd7816281f449126f2b6cfd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524475.48/warc/868587b91749d40a133a9e1c81bfbd1410f820bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524475.48/warc/8f41281ab52be2eadc8b846c032abf1162617e15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524475.48/warc/9b1ed9ca09b8015239657ad84a3e21a705107a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524502.23/warc/1dd693b6d46e8a6bbcd7f064d65b9ba1961e2c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524502.23/warc/5514cbfb187c49bb56f38fbbe14f73da593e35ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524502.23/warc/89e95fb10ad934b480d3e9a31a543a9624ec29e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524502.23/warc/97aea22d5ffe2f1b39a0d69ed3e93751a6764565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524502.23/warc/d89e2cdd89707fd34a5c28f2f0836252a5ed7dbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524503.7/warc/1623170b1ae9f2fa196f6ea001f27e3589203ab7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524503.7/warc/6c67e8be50705d5a95214d07c3128d8c5649d517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524503.7/warc/9d807596679bb1fc4aa70616120a94d8383e6231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524503.7/warc/c4ff69a0133afbbe82c44ef2b01bc1da9ce1d85f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524503.7/warc/caf4f918217891a6b9e4a828878650e92900fe6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524517.31/warc/71e6226b011abbb50af106d08fef043759b8078c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524517.31/warc/c0cb9b95ef1181353d992a6bf9f6b7ffc7202ff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524517.31/warc/c36776af888b1e10a9a05fe09c1fad17bf931a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524517.31/warc/ebe26ba2e55b7627c27d0efc1772091a4ccc710c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524522.18/warc/002dee05c706e24c6f2027596c3bad69389ce119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524522.18/warc/1bb91ace9807f5c5bd6904f1dc51edc9ab3e0b8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524522.18/warc/62615b0f15c2859c166810a8f19eab025bacc136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524522.18/warc/76dcb050ad12acc56afe0c51d7c5aec2c840cc81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524522.18/warc/dc54cc445e414c9bb213630a81cab08ff76f9c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524548.22/warc/04125ecf466ed3dbf1f4025a60af006ea71ea9fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524548.22/warc/40c31456f15a9c0bac9905eb0cd74e030bd2e4ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524548.22/warc/77b25c9ddd97bf6c815db252af49cc576f42a0b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524548.22/warc/9e4a3a46257397736f47387aea6fd045ca7cd3e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524548.22/warc/fe8a55b04ea68f587cde1789bd7df6daf2629fc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524568.14/warc/09d8ab843cd3b201340435bc4ce5c2959b4a09d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524568.14/warc/0ce0a763822879194bf313544906f615553cbacc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524568.14/warc/21c4c0fc39c1be5ffc05967e094c611fd786763b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524568.14/warc/a1ded04967de8a7cfa299b8a8c5caa195109d636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524568.14/warc/ec8c1845ed03fe3be703556adbfa7eab0ce1ee35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524679.39/warc/13aa5200efb8da602b5a9599a9a6f223b8b186a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524679.39/warc/3e539de4add1cb6e3c9a2c50029987027a3a1a74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524679.39/warc/3f3f3935916c69f8f86074e24f9475f09248f47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524679.39/warc/4b93bdfe0de1021a5bf1ace845f1853065caf02d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524679.39/warc/f5fcd7471c9fb39f78f57cd985a099f343f69679.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524685.42/warc/56f45dc5f5470fcede149cb0b40b34c4edd6e9fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524685.42/warc/97403dd544860871b776868cb86b3b15ecbd4082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524685.42/warc/99607754810711969b15ba7c443d35e2d791537f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524685.42/warc/b3f06c5c1c94c7e08fa969f01a7386f8cace0813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524879.8/warc/330f29e6b517473b159b6e488dbf0ceb7b9c9cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524879.8/warc/4666f6f4f57c42c3555572198118f37a3256f0e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524879.8/warc/8716b51614867bd372042882f952c8a91a17ff04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524879.8/warc/b3485440a6d6de642de1f913c30914579671422c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524879.8/warc/fcf67fc52f992b4fc51a2909adf0c7dc36996922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524972.66/warc/0852533d93fb6101d2b024308d35430e05fa091b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524972.66/warc/3d9033f97edd31d5fbf923823a87dc3f88ad57e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524972.66/warc/7cf9c641d79777e8af00a68ffab89bd38cbd6835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524972.66/warc/9f202a46936a8708f1a0b1733ccdadc6aac1f927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524972.66/warc/dc7247cd387894a3b43373c758fb812ca701f705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525004.24/warc/3b94b47d2ec31a8f2ba522702a9f5141e46ccff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525004.24/warc/3f8f6e46bc84a7fabcfcb346eff2e0a03307f656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525004.24/warc/56685a7775fb0a7338a6c739538a04e29cf4ceea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525004.24/warc/7be1eb0f7d6f53ff50b55f3f460a2e75145ac4b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525004.24/warc/e8490f7b1eaaf19063f48e9b2ebff65658239eab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525009.36/warc/120afe7bc9e01b3883a0675c0a5789704495e748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525009.36/warc/346f3ac5f7fc3aee30b8428cc8d4518db6c5a85f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525009.36/warc/799e0fc3096334c5ac28efa93e8a2dceebc6ee0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525009.36/warc/c2f677978595a3e683cac9bebf6dab16c57c4652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525009.36/warc/f96e18162bd76a400f939f0eae8d0823bcdd9d5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525046.5/warc/09b51857a8a40280690aa44deae140189a6f79a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525046.5/warc/23ab004ea39f8e6a8bcda9362c4b6b297d47f19b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525046.5/warc/30658a5e5d5e4cb050bf818bf003e5050388b20b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525046.5/warc/e04a23207b1a45bd4009c23e125af30814611ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525094.53/warc/1a31727d64c8e76cd98900d72e64d834e5cf031d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525094.53/warc/1afe5aa83919596b34cb22e51d32f81efbad2e10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525094.53/warc/3ddab9bf8c98cbcb9558c29afec4a74de3b0f596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525094.53/warc/8c6ac684cd137aea4c6c97ee515a389319f63c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525094.53/warc/a52c940a9f7f3bae2ad699240fc1e7f9ad4a834d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525133.20/warc/25f062ad9403602e75dbfceb1e2cf334f7bb3448.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525133.20/warc/5963b73035e95b38e9d511061f2c49f51bdee21f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525133.20/warc/5ae1c0218cacb8258af6b8484a9814b6d28f9895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525133.20/warc/ec27f62e97c79b60cd772bea1cd14903d618b87a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525133.20/warc/f152b639fbdba77b7554b5b9854aae0fac01f2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525136.58/warc/28d3da943ccd72ba7e756a60ced473c84f151e5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525136.58/warc/57b6361df7c6339f21ffefa47cb036b684f49679.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525136.58/warc/61ce7f8a175092b451001fb749c0415570ea5b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525136.58/warc/bd30b4d87446621f76d006b4a5571faad05f972b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525136.58/warc/f5c06c37aec8fab5b58af1fd5928fd32e824451a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525187.9/warc/6a716ed80b6f7db6bf39eacf2fbb3e9abd3d789e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525187.9/warc/6f24ae35287df5977bfa685d4069591b77927c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525187.9/warc/7c795f90181c9db8a31bd13d44ee4c4e98766532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525187.9/warc/c03b2cea570a1e9965b0e4a8f950d473721399b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525187.9/warc/ea7a2cc2d8ae3dced8c31cda901d9dd0555bb24e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525312.3/warc/32efd01810757c015eecb0131f15d96ddb5c0318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525312.3/warc/b6bcfe731baf1ee15a0f5066494f6f70a5da6fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525312.3/warc/c354039be88922f671017bc56e94949dbb8bc14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525312.3/warc/eb7cf32fb2791a221055a79429726849af9bdf39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525355.54/warc/2450c564d475a5ec43e4fde73b3072c697b7fad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525355.54/warc/2fe3ecea11d16e0d3863e67614323acb14dbb821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525355.54/warc/7d3a8a43fc6ea30011fd42bbf6ece00a636caf01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525355.54/warc/bf7f91999a60736d02b6e83406540cd72a06c4f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525355.54/warc/f6d59fca6f01c8cf74855ae8d3b777f3daa22d59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525374.43/warc/2922f85118b1e72dce092c72f9b13a84c4a83daa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525374.43/warc/82896904d449977542ef31bf410f8c5cdd8cc680.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525374.43/warc/878d8739961c2e99c6252aa629332da1ce6a5366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525374.43/warc/9aea040dd5432b40f6c817bb02faf4a18798f472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525374.43/warc/ed9a7dbb2aa94bebc578a5ac1dcb1b6ea9b4919e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525402.30/warc/0b55ea83e0abc6fb860345f8cdfa92bd92914ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525402.30/warc/0f83b9003ea1d91825a50bdfebfe61ff28dc7aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525402.30/warc/703d103fdc2133d9d974315e1316d6d97ca6ff56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525402.30/warc/756c57e74d41c937b14b908c3db9e6279d8a3ceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525402.30/warc/c7641ede7762b6fe33f09978578ea205335bf366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525414.52/warc/0e35e385eb3a08067e3994a9ee33e307eb161072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525414.52/warc/8aa8154836bc9aa100cebed91005c63981e85062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525414.52/warc/bdf4ed4187d6f107f8294fdf16d96a18fe3ededa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525414.52/warc/c3f7225af1f18165592ee5cca25988fa6dac1b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525414.52/warc/f35479f043d203d19243d75e2b9eee110ad1e596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.62/warc/2067f9bb38bdf091d901ce62c34ac3272070e99c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.62/warc/9a08aeaeda8155e8e8b9bbf08eab6cd22487c49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.62/warc/a1f5720c012b2303d92178a6dffe98e44d0e3a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.62/warc/cbe567ec54e1d8fcc77f00b99186b4af709cc86c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.64/warc/750af18d4f68ee6757ff508db2008f94078cc222.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.64/warc/c6329fb01a810480d050f8f4cf75d431f5902a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.64/warc/cfda8aa624389f4f8aebbc8adbfb741d95c3c2ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.64/warc/debcb7e4246feb217f8d72b77deaae345f2eef77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.64/warc/e2641bffb4ee47da9359a96825d4e73c235a0c75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525500.21/warc/0941fe6cd8f7554f4b1df5fdfc9ef14186db476d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525500.21/warc/14018471b2e49b6d1fdc305ee2ec62259c25ee5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525500.21/warc/303d43f97890b07dfb4f5b1bb855c1e484702ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525500.21/warc/a43dc2163e9d67d292c392edb7cf964820e1ef4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525500.21/warc/c771f93b989c4f0a188a60d8aeb6ea1e7c8e0a3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525524.12/warc/73dddc59b04dc3db99b295be41960ee87e289a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525524.12/warc/97b879db0d769b654abc4dc8fc91b13aa7125e58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525524.12/warc/b210193cc24a881e0d14402ef06963d7794b9d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525524.12/warc/e31ef42aba339fb4ac2c39674f939acd8f3871b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525524.12/warc/ed439aa4838c9373689a4367c58c7eff4cb48af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525587.2/warc/106d58832afcfdd234503284d7f98adb40020c85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525587.2/warc/2c32ce6059750fcc7c3e2abe36abb44482e9f28e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525587.2/warc/305ed3483938d3e9b9633218a8afe8b4f5e4e0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525587.2/warc/4ec25910a267af8d2b5327ef22c381068e703fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525587.2/warc/e06fe51359f6a2d9f66a09aa56f0502fa9022948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525627.38/warc/1b86cf987888bf8cb9095692634524860df75055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525627.38/warc/2c63636ba9e9fb6f35f938c95948e728f18e40a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525627.38/warc/429e7537ed5c6eaf26696a00bff0a228b8af018e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525627.38/warc/476cc4971e5dffd9eebee2abf4761327a6980c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525634.13/warc/030e0c9344963ec1d0fb89aec3ed47b33ed2db8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525634.13/warc/3e8011c95117ef86ca1def4455704d2497401e77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525634.13/warc/6894936eea3929ea146648e8f7140d5a0d3d2e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525634.13/warc/d0881819e79a5503a28836120b21af157dfbb300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525634.13/warc/e2b28a5824482a93c8fb931dfea9e16a23a761a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525659.27/warc/1c79a3039d91b9c417913aae200eb02a513e3faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525659.27/warc/7167c63e25445b0c62da758ac08cceeb5c3dc097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525659.27/warc/73c2c0980893ea0cc637fe7cd2e22bdc4200ada3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525659.27/warc/9e79ada5c7e04f49c9030dfd3a36350c8d0dbf6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525659.27/warc/ac42152304fca3fbbe7b7a77dea8bef7e42198ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525699.51/warc/00d6a2aee83386d1f053a5cd57b3dcfa8d01fa0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525699.51/warc/541bc0c8d568727b5f278463dda0d14582dca630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525699.51/warc/7abb91241ca5b56585900beeef0531fe7313a984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525699.51/warc/a4aae3c058290a02e0a894d07ed932e248eb12ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525699.51/warc/c5a67a9f274323a6ebdb0958b9b1f1602df43098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525793.19/warc/34e1fc2c10779626e7c35980529dc8c19731acaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525793.19/warc/512f0be2f348c07577ec016761bec875fa953178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525793.19/warc/b4d56fe26b318dc7aa2a612cd6ed5af44c2903c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525793.19/warc/d5d42f2c45e146cd22dbe396b3f58e35c0271aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525793.19/warc/d85989d72351b0b69c16a289d6989b38bf4ff636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525829.33/warc/636ecdfe7fb343540c1126b3fb1977fccf104c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525829.33/warc/6b27f37c42ba4571e277cb6bca86610d47a30bb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525829.33/warc/aca67fb231c3385317a5033f6060ae985353aaa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525829.33/warc/ec36903a8e60820eec6799fbffcc61d5626b798c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525863.49/warc/1e479f47a692feb9921aa8fd70ba9df8c9df9d5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525863.49/warc/489fedae0fcd2fd1f7b9d4acfd4a2e1c01cd6cce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525863.49/warc/49b64790e399e892e743523f516764d303ff3d11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525863.49/warc/98d7a5d7550f06625f7143163ace7e7522b6fab5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525863.49/warc/b984602fac69f8ebe6c30681649aa3c2e530e27e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525973.56/warc/197e05fe2c2c5c5e8ae23d6d2c53ef450d7c609c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525973.56/warc/2758a63ff6c8ae16790594ad7e1cd31f118de345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525973.56/warc/8f69d9ac391f9c6a781b185ccf59c75f2a834e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525973.56/warc/c703312ffa979be48563740ac4176bb7dff78ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525973.56/warc/e12c504f2f71f9f647f3219ee9be79d0f8139040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525974.74/warc/ad41c0a0945c89e2561b89da06192982c18ccea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525974.74/warc/b5b42a383a9d589a6d8f068df52bab2a14d18736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525974.74/warc/bb7534fd9c3ea1e92a1744a48f76dd6908ba0518.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525974.74/warc/cee97e15f07a8d14aae506ad014696b3f4d28428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525974.74/warc/d37a9f530a87e1e2313a300c668de52ad8fc1107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526064.11/warc/00d52525a23924361df1a7904a92887b276c5f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526064.11/warc/5309ac208f9dffcbf239a61703ed0e3f00955b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526064.11/warc/80a82a4138ebc9714d52524a6a00203edb441e95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526064.11/warc/9aabd453b9c16e4b0d992fd578fe639a858e1c32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526064.11/warc/ccb8d59a6664e41ad74f6a4ae89efb4982c0d8f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526153.35/warc/19c7b79b6323cd9b1e3738e531ae323f114ed8e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526153.35/warc/2c71726ae700c2ae5a1940078fd1ee760085537e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526153.35/warc/3c5611e573f845ce5ab84f86942fe381d5fefc11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526153.35/warc/b572c09351143350414cd233f41b5e57308e4489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526210.32/warc/3475e402f786f52dd9cea27ea570608a0e91333f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526210.32/warc/90d0a0b242c4cca0342bc20f77caca4b53fa4669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526210.32/warc/ba99662cdd21952ebe42ddf1acf7382c3c4586ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526210.32/warc/e286668386371bddb9b376685096d35df006e74b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526210.32/warc/e3b13b747ceb0a9e0de82b428bb3a25da6218aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526237.47/warc/51fa5f50ce35e83e4be77e5a7a31279e3703b4a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526237.47/warc/b5f261c1a1fbf60dc3b5b800d347fe9344db186d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526237.47/warc/eb6fc748008d2884c5e8a4698436aafe938247c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526237.47/warc/f8ad679cbf6523a19f34e55f46aeb9d53f4dbdc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526237.47/warc/f916b648ca50fba60dc8b5169040f74979564ceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526254.26/warc/23889f926614acffb27625827dbed99771d34811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526254.26/warc/28a76154790f9fcdd3f1eb2568b0a8623f78138d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526254.26/warc/6f2454dc2fadd9ae650869364d05566fed9f66ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526254.26/warc/99cba2a96c68e21872b96594b29e7e770a4ca47a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526254.26/warc/9e6b8667e24cb2949a806ebfb050623d395bd7bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526324.57/warc/119ceef143da6760df1e975cb318af5d84282ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526324.57/warc/7dbee3a8cc7074abcb9b34cb62f296a4c614f75d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526324.57/warc/8ec059ab36450cd86b10a2c3a1d172f2891b1026.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526324.57/warc/8f4a13cf4580df80067efa8def424ca0d8a76209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526324.57/warc/a9f86079e6a67d55410c7122c7e1d7ae098198dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526337.45/warc/597e6a0a80298c0069aae40bb4bcd122a2bc8f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526337.45/warc/94b20fc6a3b0e22f8d4fae1c489af995ee6b0a2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526337.45/warc/9da101bc5ff8119112bebd21ce453283336286f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526337.45/warc/bb5522ea062faf92cd83b31b0dd76d413f7d58a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526359.16/warc/0023bb32b7b0d86c1b285901054bef6ee6835cc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526359.16/warc/35314b3a59148e169f207952bef0d8a1b24da90e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526359.16/warc/38d61fb83f148404eee0f466f9ef3b7c72400c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526359.16/warc/3cc45027f6e069822355bf19777b533e12f54a23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526359.16/warc/8458bd31534b54aef56247dc468cd25a976e4a82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526386.37/warc/0c2ade094cddeda87adc30ee91fe862968e148d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526386.37/warc/1b4dbce2e60eee57ed380335b621f014c808ef46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526386.37/warc/444e2f50c0cd2dc3f0d074b481f9068d8f1208e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526386.37/warc/7314717aced719416057e4e276ea64c317a5d7c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526386.37/warc/e286389e8d3910ecb2f9e3bf9547dc45a32a6ad8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526401.41/warc/2703fdba702ef27cf1aad6841889823f047527e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526401.41/warc/33b28e4384a4728a24fa4cf73a8d53e18d219490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526401.41/warc/4d9106d92b60f5340e49416d711eab14a328cf96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526401.41/warc/7f3efbd622b3aed9fa2c1ec693772928513e4387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526401.41/warc/c6951c7804a34e557e21c55f0687c5fd9b5abb1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526408.59/warc/08e5ff22e2281161ccf601446423a74f0a57d1eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526408.59/warc/11912bbdec4b1487e6bf328f47278f93825fecf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526408.59/warc/15b9339f5f8556989515c274ce2408df3e6863dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526408.59/warc/af920710c55f67839e908eb91454a3db1f95055b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526408.59/warc/f8b483421401f011895cf538fa62c29d3cf68893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526446.61/warc/0a623eaf0cd49bb86f16fcc6d6650909aa08a33e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526446.61/warc/4b857e4ccded15d5771b8aac67c66e4f11272607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526446.61/warc/c5446dc514a9c4290430e6c33223435ce1268206.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526446.61/warc/c5f0dd7fad04435d46dc8bbca525b37916ab0350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526489.6/warc/04f7add3a7ca9d557d352cf0129d7b3ee0b9848f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526489.6/warc/0f98e2562f5d6a568016b9d71806e981271bf8d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526489.6/warc/a6ff6ee56ffff26f0bb799df0ad6c7939bb45d08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526489.6/warc/c4e8780d016a5018e7f77af812a47cfcb783e656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526489.6/warc/e214578d9339b915ded03c2db2d75666f02aab85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526506.44/warc/21d5ece9c122ad59d039469625afcaf686c97a3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526506.44/warc/468275310fa9db84af6b5c137f02c57ffdeac2f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526506.44/warc/85909f8da4521325985f84311211da543f4b4b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526506.44/warc/a8133a9cb68531c774ac628b9937d065d5448a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526506.44/warc/c9396fbefab23430c9183ebb17a4a803f8e90ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526508.29/warc/06da99be9adb050cd4731ef6f96ac435ccc28bcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526508.29/warc/371ac3bc681b3feef64663ee4441ef759e3fd39a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526508.29/warc/48fd1fedfc152fe3174cba0096782025673a1fc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526508.29/warc/7a46a953c4bf04f5d601d5c23eef6a2dbd4e1c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526508.29/warc/b4e13cb38a60d4ad09c6292a88f19f7f17ff0642.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526517.67/warc/5c1c20155c8cc7fdf0eb62d8a8f16f34f5d9538b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526517.67/warc/63e403856a32c70e8c62b6e2f8f6d27aa14c0714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526517.67/warc/7de2eed65558f1fb003b2732a067774e71580ce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526517.67/warc/a6dd67b11a9b9fa03213f0fd7f84367fe25a73ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526517.67/warc/adf55b34b200f574c82a28b13043d6324eb6bf7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526536.46/warc/4345b3e67035bb7a0ce784d8c205b449b9ad6b4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526536.46/warc/96b6a8e52e6dd40c429f5817094d765bd3d39f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526536.46/warc/e267b8f802171c69d98ca6b204008fec6637bf6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526536.46/warc/ed0a0296753f436257b72c27351b7517d1dc0bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526560.40/warc/1be792b2362f0e9d2b427ebc6cb44f393db19df9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526560.40/warc/5bf2fb3efdf2bafeb4e275dab69a950fb777f4f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526560.40/warc/a988dda9e82a402ad183a059dd47c9e54367098f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526560.40/warc/e9f276bf0e2445fa50f6f89138fb5ac12954b26b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526560.40/warc/edc67559fa9fdcaa1583e4c4b289e53879d9ba4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526670.1/warc/09a9909aff9035195ee167c1377b92e1ed090795.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526670.1/warc/73e85757681ac8b9c28acc3a3bd4202e42ef697d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526670.1/warc/b5869c758fc9592491532fc7e4781c19b7d22f03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526670.1/warc/f877ddfdec59f2f32383845efc6cd25f6503f2ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526670.1/warc/fb707d3777c137030ce0c71d6da3ae46dca6c919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526714.15/warc/1601782102e2ea5d9c108c4456b10b9a447b8389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526714.15/warc/668f477d208e83a1f8de17afab37f5b35c93e4dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526714.15/warc/7ceff9e3a03d7f1840a13f40198b591190a0001f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526714.15/warc/9a36738030d40f4968b442e9b2bfbe76a4c51cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526714.15/warc/eaae60ccd0797399da8b476a6b201fe046e75c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526799.4/warc/08b67bd921e4692956dc735f4ed50a29bd747ac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526799.4/warc/9b2fee22ca900ae6cc413d306b551382dcdbc4b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526799.4/warc/ad519a2e88813864223929b99411872bc46414ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526799.4/warc/b3e1fe3a08a064f7d58015cdd4ad20a9598ca919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526799.4/warc/d88c67872bd9d58e406f24e82429e0f1e5db235c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526818.17/warc/3fa32f9887ea8c3f4dc1fd130ff175d306ce39e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526818.17/warc/66c49c88a1df8ea9e315bbe5f20743125b9bb132.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526818.17/warc/91bec5a4264529dc9138fb395501e924d66167a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526818.17/warc/b37bcc42a9fecf525ac93f8229ded1a73d32f726.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526888.75/warc/38ae1688c7134c43c12fb09cf7c0b096bc2c2e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526888.75/warc/45f3ca02095426d07051620e926aa027055c0c0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526888.75/warc/5db00e19be46499b37bd702261f14f1d8fe1aa8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526888.75/warc/ba81872d79fdfae4ed57f00e8658ae14a806df7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526888.75/warc/c3d762bc35ac251b99aeb20919a4746b906fecc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526931.25/warc/4d72644cfe6121a2d83b5682e48239ab3ea57973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526931.25/warc/573dc18e4fd658db094c9415b14b1afb7ba1c870.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526931.25/warc/7d5731fe6d393c17119f0c4f3a29c94e6cc50b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526931.25/warc/7e5dc100cdfad151f38b9b1e1543d8499d1c37a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526931.25/warc/bfad58b88a39007d34cc050c16ae518861cc2cc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526940.0/warc/286f04a3995a359de09934b837e3140d337b9d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526940.0/warc/54e7452458f7e872b467eb4e3a7cf16e30139c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526940.0/warc/c00ee35401edcf536b43a4b0be39126d63fa5ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526940.0/warc/c8e746297a7bf5a1336e615b3a63c012367060b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526940.0/warc/e8eedc1fbc0c3ac74bd987e34d4e9512bb94cf8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526948.55/warc/12e3efd8dbdb644333263acbd074aaf8314769d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526948.55/warc/252b6fae1b26e59f462e6ceb3bbed09119aac71f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526948.55/warc/71f1559c013a149928fcb8ac9939617e912d8886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526948.55/warc/bb8496b0feadfc9e14730e11617375e2e578aede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526948.55/warc/c260e0f4f278e5a02ccb882c5792aee1c9cdbfa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527000.10/warc/08c2c18701e1d2c310caea1a0cc02acebafdbd24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527000.10/warc/202e2da4c8d779a041a0e3b9d54c8e5c31aeffb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527000.10/warc/53da2100e6bc6c408e79dad54d6ac57bd018f71d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527000.10/warc/bde1c96f01bccd09e2cb9e2b8e38836c1c0f248e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527048.80/warc/48298b9bb68442d8f11830edf390fe5942fd5985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527048.80/warc/920383e9c4d2473d10ea61d6f95796f666afab62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527048.80/warc/ae319ca84e571c319fe7ba93157e5a86764b0f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527048.80/warc/daa2e9aed93cadbcc2fb810086f2cc26a3ae9a40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527048.80/warc/f981f5b1798659d9e15a4dabed0dfbbf25966121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527089.77/warc/4eff0952023e2f6b421615a1328428515cbdf8ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527089.77/warc/930434d19dcf1ca4d972e7d577175a5fcb9bb082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527089.77/warc/bb7a156408690c8e86854f11bf92821a7521085e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527089.77/warc/c6238f86cb00ee44f9a97c0feb940d2c8dd64a2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527089.77/warc/c95528afee76c9a383bf934ea738a2457f5a5304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527196.68/warc/24eb58499565a51ac401479a602e3fb90c9f487c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527196.68/warc/98da10844dfd0880637a19e298f6702aa5162d71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527196.68/warc/b730bf6f57f0a654f700336efd21ec729b0fdb3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527196.68/warc/ddaa4ffe0011300675c02e206922785897552ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527196.68/warc/f132a130403f332762de0e8728ca0b050e21ddc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527204.71/warc/545470103573482218834e85bb7725785b078105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527204.71/warc/9810f415df6437a8317ea5b0fb8ec3dbad26e817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527204.71/warc/b4542416de5a914b9d1255defbf3780fdcea5db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527204.71/warc/e3ebc378a7399323b50ca0aee8a949f88625d963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527204.71/warc/e58dfc4643f70da500539825776b6f14530edd6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527396.78/warc/641067c478a74d1ff6b21ef946984a55df93cc07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527396.78/warc/806cc44297923e4628669c2ce0a4665e4e4d84df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527396.78/warc/abacf486148779eec46f4ca2af30cebae265a93e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527396.78/warc/de98eca5dd1fdf18755912efae7a7c6556183c9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527458.86/warc/03f76a6fcf87b9d22b19bf63e24c977fcf9647f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527458.86/warc/31fbf1878431ae6f6d074c5d50b5bc8cdf32b6b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527458.86/warc/3dce4ca06f33429a446ca3ff14f72b5b50cfbb65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527458.86/warc/6dcc82a9e2ec3eee1114c1d8f08c5e8fc470e550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527458.86/warc/76a88cdcd99d3406732a8e61a0b8303ba9a657ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527474.85/warc/71ce9ad0ce45c0a319760ddadfc4ff27e6f87cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527474.85/warc/af356c07f7eaad459a74e1fd7771cd71ff804b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527474.85/warc/c5cd6434692c3acb24458d9c6d978fd788e25af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527474.85/warc/d0a1e7db620cb30a5b43e751c6d801491517b047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527474.85/warc/d22afcc7c8d489c1ca869efd360b3e3aae81b139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527531.84/warc/443a34146fa772571f0e5c111fb53824a75628bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527531.84/warc/6575c9f353ddf928ed04ef140eea267d0687b1ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527531.84/warc/9d896dc213c1c59421df9d755d517234834c896d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527531.84/warc/ad9c177d599bf3955fc48ff9b3569365ccf1338e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527531.84/warc/fa08c1fc19f87683181dc0c2ca1595f13def1784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527828.69/warc/151acce261115eec8b4736b35e226e3fe291ba60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527828.69/warc/461f3194bc6998e88275d9402f4ef866530e97d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527828.69/warc/7912f70b742e1fa1a5221c4b63c789eb19527c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527828.69/warc/dd17c8bda79d4a5e057e9479e915832eee18cd1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527828.69/warc/dfd7d9be766d673cbc4ab4f96eee727d41cfd86e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527907.70/warc/258fa2bc9d25f3f36236c8c8d84c738e091b89b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527907.70/warc/25967a08fa635443f8b69d703088ab8d9b00a165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527907.70/warc/6875405141524a6afe66598fde53033da73411ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527907.70/warc/bff2227370dd4f1d7e4e3d22852c5ac1821c568c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528013.81/warc/11c87974f509a29f11a6ffd9f3a04dca0d46f894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528013.81/warc/395f61408e4d6ad5239afc47acc1d7e8a12e3128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528013.81/warc/793e0c967666d9541cacf65909a084a2fa47f71c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528013.81/warc/902010193b0562a409463a14c4c5933efc1632e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528013.81/warc/dc434e8a1f7f8ca8510c7747eeac9efeaf80c431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528037.92/warc/1dc0387ec37cde4d1f1197d55e5fbe01c0d00a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528037.92/warc/7768820dd94fd2a8c9a3a6bec3d28359abb3f5cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528037.92/warc/917805d9a561334ffd45a697a52260bc1018fbad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528037.92/warc/eedecc926710e21dcc4b0983537b4d450d650a1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528037.92/warc/fc0589d466a90b3f73284a8b2268239aa9f19160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528141.87/warc/5ed7c362f6d770e49caad4f4e5a6d11d5f703bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528141.87/warc/8b7bb29c4a72f4c164deb429195f99150a164127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528141.87/warc/bacdc66420439b55300dcaec41b76f0d8e35a491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528141.87/warc/c0b68a9c09041e1fa5927f882589faf5f01e983c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528141.87/warc/d2fb9ff2d6f1374595ea5dd5bba7cac3142b08bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528208.76/warc/0ed61f91fc1c3e5f7e2342b81ed2454c2cb347b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528208.76/warc/16e1f571984d809697a672cdba70e5e926769dbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528208.76/warc/6485d2d59634434116d9e8c4e3f0353da801355b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528208.76/warc/e42c0e6dd960208b84482a32b54a5a6ca9727a84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528208.76/warc/fd9b16e2014d3801c2b0e5b83c2583c43f720dee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528220.95/warc/0fc0291668bdf87bf01ec0830f1da3750e79f792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528220.95/warc/121bc1aca76588de95fade608d5c0f6cec40864f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528220.95/warc/16bff081f686ce5b33c1034d764d7c12630d4350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528220.95/warc/77f69e6dafc765896df66a10fe53c1344484f00e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528290.72/warc/34407ae8ae444bfa76c3dc22097de714db0b306e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528290.72/warc/48a38b1b4d0d5511c35d85313cac734a2bdb238e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528290.72/warc/67b9cee104a7378e795499096589fd770a73919c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528290.72/warc/76ecc1fc607e7f529dc22162e1205d4b8c1ac6d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528290.72/warc/8df8202f3caacdae5d385a5966372ab0c722552c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528635.94/warc/3f253d000d7c2f3814100a44598a83498c14592c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528635.94/warc/5b6e1a031658ef74d0246b0f23c2104271783ac1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528635.94/warc/d3b15a310a8415d15ddaf1e35db2061604e1bfec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528635.94/warc/db1d66429ad487f0782fdd37b148dad88832a502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528635.94/warc/e4d7826f5d3fd6049d64b86fb97696232e15a239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528687.63/warc/0918bba461cfa80bcd39b9f2d095578d6ac9a442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528687.63/warc/3cfdc6fcc4d902c82ff2b7de5391947ae0f41c61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528687.63/warc/63134306b67c59a7925b3a5c73f1aacb305bd273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528687.63/warc/7eba27cf328de16d5bcf77e088e49f353baf5d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528687.63/warc/9c559b6daa4f5707a35d64056ac7fbe648561fba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528869.90/warc/2e7e9a5aabe790b2d73b5c3a15011f4a21ad3662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528869.90/warc/79619f38c973a88f4037679a7b8706961f691ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528869.90/warc/bf2365c8ea74a8e2d4ec6e97f1604bc6185010f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528869.90/warc/c2ea12b6bb5576629f63cb620418677d5c1cb6cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528869.90/warc/eb9c6c62e0248d41a96843874a0671a785caf49c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529007.88/warc/0e3c07e687831e85af99b066de3e557fef3bbfdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529007.88/warc/17c50519ec4b520422be648eba7c85c13ab4b5e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529007.88/warc/37ab8aeff58fb7e26629fc092af8b1c94cbbc864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529007.88/warc/a6538d31f95e058adc5dc0f886a73141470651af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529175.83/warc/21c12296524b96870a86266996c1e2bcc60720ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529175.83/warc/77276b7db2c21579fdafb71d8b3c986e364843b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529175.83/warc/aa08f43c1c6bd8b319fdcdbf358847598dc7e991.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529175.83/warc/ac495ddedae31d187fb13dd80a0e9de6e42868a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529175.83/warc/d0549a347fba68f5e20b162ddf60b12a724efe6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529276.65/warc/095aee48745dc3d15c12c4173d83344b0b023570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529276.65/warc/5aba847fff28bff40063a88bdfbb4ce6b6343a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529276.65/warc/90761a9e9f04ec3c0093ec9fa31fe6a1fa711811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529276.65/warc/b49da325c3971cdb4728e509bcf638b4ab4c39a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529276.65/warc/d59ae8c67acd8f08a189c21c8674c9d7f4696329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529406.97/warc/358458ec03627a387868cad2eadab41eb9f74de0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529406.97/warc/490608345d492095476851b72b26c5383a49edca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529406.97/warc/a73030b7218cec09f321cc2c594859cce92c5f35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529406.97/warc/da4f435f767ff2354b1e41585dbe9cd9519395d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529406.97/warc/f9899a98f89e1f087649f2d409dfad0e80d8a89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529480.89/warc/26921454b162a259c9d8b98270fcae9a49ebf93f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529480.89/warc/990bd5f918e9108625b4d47428f80d8f42bd8081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529480.89/warc/9d6df5f34f1d614721e4385cea8355554fb90a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529480.89/warc/e433c06cde3c31654c828e671fac9ff78668ebe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529480.89/warc/f7e79782992ccb9a368e4a14c5c62afc31e90eac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529481.73/warc/1b8cd0d4d6d6643351afe3ca172270df18312a7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529481.73/warc/717e28beee0b869e5e70da7dea534dacf20ef964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529481.73/warc/e54d7ab2519a73cd2cd59d6cfc50afa06a6f7d5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529481.73/warc/f32bc40db4bc4c18bfbf76056d95de70379521a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529664.96/warc/5baa99fa84d5345cded85eba56a1e4f4434e09f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529664.96/warc/5eeecb24ee29975dd3fbf67bd95a64658cdb3bc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529664.96/warc/76f6f008105e3ea62963670d34bbbeda8d0fd26d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529664.96/warc/be6c074ba18f70fd225940ae188301e142926878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529664.96/warc/dcb53e906ab8e5829f356dce86e159f23bb07dde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529737.79/warc/417ea9e1241722c2045954e77515e010a3e599d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529737.79/warc/48badcc358a5cbfa2071243f5c6f27a35ac8b70d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529737.79/warc/5d7a0b4fcb2ca4be02b08366401db496c7bd4cbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529737.79/warc/68bd3258a1528ca44cd91ab034f07ef1dc6bafe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529737.79/warc/79d3ee084024dea1e33c84d65a2b1dffa49ca87b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530246.91/warc/51202b45edf76dd69f5565ea6a927fba5b326759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530246.91/warc/6a4e7a02d0ede5b4368e5eb9587c626a77b85da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530246.91/warc/a6919da1061705a4386bfc4d12f7d86efa478b89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530246.91/warc/d05518148db4558301719a23130ba301d2065eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530246.91/warc/ffa2f282b03c6f4177a0d57d4584b61355a0621a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530250.98/warc/1be6fbebb5d8d4199d98403b05654a21b53bcea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530250.98/warc/85ef1a5c8d386b4266cd5af31f67c070b5652fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530250.98/warc/ccc968bdf931c586ea6597ca3cf51d4d98484c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530250.98/warc/deb8582b40a2cce09d0dd8cabeb09c26c2e45538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530250.98/warc/f613df4e013433099804165ebb4e408b925d4952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530385.82/warc/37e41f35978c7c854ed2cf6bf512c6f8ed4c80d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530385.82/warc/47ab7bcf4d38d32830221bd3c1460c7954801e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530385.82/warc/8b331f8252c2b39e16fdebe15dfe37581ae10fe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530385.82/warc/c097d13586b9dc03975fc7fedff1ea590fbb5733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195531106.93/warc/2561d27451e3b2e9b94d90e77249f1e530d2085a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195531106.93/warc/2c8f4adfaac88bb591b7397ba6262496b1b9587b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195531106.93/warc/3696158ca81f22b71bbc163a76d5cb797bfde47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195531106.93/warc/df9988516354d1bd94da61c07265d57a6e03328c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195531106.93/warc/f829962db58cd9d7e8de9787f7903808c0c1719c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195532251.99/warc/2c8e932c69a1a95314331164bc726270568a89d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195532251.99/warc/3c1b302ff226dd74b3a306895a032190228aa4d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195532251.99/warc/b67378435bdce776d2f87d33710dc681b74c0cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195532251.99/warc/b97b2708194d5e4543c157f12bbe6c9736174ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195532251.99/warc/edcab92625a7f184c344d70a7c4cd982fcce5005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1af3f49a89f0551b4a1f20b209f8c5f295878f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/2118002603d6be139e6f989423055accf8c5a5fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/2497c4cbbd7eefd9e3b77755ce6afab010efabfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/25e50c76fa18446843f317fcdacc524d8cc7082f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/26f755b138946b952981fed343166c0584fd2156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/2bc64926a1b664607a617c76944433d2b3efad24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/30845028c60b257b45d61c12574fb0bcadcffef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/31e482c13d5e938a96a41324f40b5252649fe568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/32e7245a68f1d0a22783650853facdf3dc485b56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/339fed8cb0e9d065ffdc827dbe220fc1bfc1f84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/34977ebd28d65d91d4f45e6772b9bc19d7c3f5ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/360d91ccbba6c87235139f0327e0b6604a3c55e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/3b8e4e02b58cb8520fbf211d1c5e597b6ab722c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/404df100a3d6d386bfc267c8a96e4c5ef2106332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/49166efe9dcb659f55367a4ec9523f670fc3ab30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/49cb37ae670964e061f54042f01623fc5f66fdaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/4a6aea2df3b52ca0563319661e95c7514063a768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/4d6f938e0b6bdc754f0ba47c3488c6ac966f42f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/4f5515d94c390cfa6167a3ba39aaa444b16dfe08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/5116bd8891271b8cc9e497cf33e4dd37de06b604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/551d788b0105de3b3d21b3780e65ff169286f432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/56f3e39c3f889bc4e2cb0360183a7751b569e551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/594868336d89d1017d022f01a231b7a348e01416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/5b22a3c6a515c455c3225a88f790d286c7f95c80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/606003e94f562cc605a548c5b8389e79fa2f97e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/63b75178669b54a11cb731d27ab0cc36aacd6da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/665a104ecfe8a1219adaed8c661200576d251a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/6ce59e6354de2068ad4a2c748703f2046f7aff09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/6e736755a75c8c9a552fc952bbac8447e86f4251.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/6fd578a49b2df4a9526c219ad65b6bf6c6c8235c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/70589026c97b1b49877320a14939f284805a5604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/71ca598d2b362286f8fa4af15c8ee0859cba6e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/75c811e5420750f0d14f2f456117a1d99326b26f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/78c7ba37ff1b1e333953f948db48811c3cd838a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/7b6e6ed97c47e129d33bf6b4022d8496a2dd84ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/85f9454ea9a65dd7feacd0a14b56ffc06cad8eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/864abeefddbb5e2bc6e3d951741cc0484ef28d3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/866a4d2df4bfa11b731bb9c17a9ef6992ec5ab4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/87ba26654dc173ca5ac961b5cc990a33b7bf6f3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/8bc6a5ce44b2d986aa8a4bc22f24401667c3923e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/8c5557ffd856ce23dd1eb177ecb9ceb2f21ae599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/91cf1b8816e7b4f19785e8c8185f0b0e5586ab1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/9a1b096090a1f627309304ba4d9cbc9cc3cd3da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/9be2e811ef5428c3ea92c47217cfc3e1eaaee7f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/9c3a8f77d19445e59f38eae0826cc459cf7f4c2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/9e87db1b3d12471c466d94ec98b4274fa6840a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/a17cf2703669bb141a528b6ea0bb94ba87cf6e63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/a7ffe60c4aaf6df347e60a5ea8c8a9bfcfaefac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/ad1d8ac6802e582477fa811c03df48a57030668d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/af1f6ef38b3dedfc77cba29c91fc56bbe7c9997c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/aff7401cf350c0dae67d4bca56ae672182c46e3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/b4a6bcccfa42cac5344fe5813e01af8669fa0791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/b4dade8801a7e0803052f4c358d51ebf17091305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/b9b323d5e15c200c30215b4348f07a5c33c911b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/bca0026af998f16cce54653aaa12ddaafa2b2369.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/c45e09ef278a5a5a2720348b37764b6947c8ee27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/c9737bdedeb1322c71496425cb8882b505be3181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/cbc3f619c36f56786edcd9d8ca8fe36519188a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/cf1b4213aabf8349daeda94866532f077d68294c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/dc0a5c0976517504e6384b2601ab585fa62f71d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/e0078b0c1fbbc105d97ca6b1ac45729d836a5e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/ee8aa1f353ae2ea5ebf8afb08c6267793d659097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/eea680c27cc2e54378aa8f7b05f0d763fadfeb5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/f535e02273079552192770fd275c53172e7e3d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/fb13ff6f0ddda858b853de365166c27231011a17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/fe0e09b53d890e5d82064b1c216a56944a7b804e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/ff2441d6895526be54fb6825bcb836e9109facd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/ff5a46fb588689c2c1c6b95d415cfd0312d5ba3f.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2019-30/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2019-30/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2019-30/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2019-35.yaml b/configs/cccc/dedupe-url/CC-MAIN-2019-35.yaml new file mode 100644 index 00000000..010a6fd4 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2019-35.yaml @@ -0,0 +1,581 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/023577e73189c373dbbf831711d04eb43a27893d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/048a320b5f2ca6181a219c039f66141ee6968ac1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/08d106ff1c62b580fd69a05ccc886dd6c771c432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/09b0af8ff2f88f756ca1b1bf2d7a8b080fdb9053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/0b427afde0da636f1c765c1bb8d0c644d8162a7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/0e29dc9f05c9cd9a7db5b6ffba7678de8d9f7f43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/0fd04f40035971dcdbeb6e6fec3a5b1511bccf29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/11ddae3e014819341f39e2c64918c2338a9b1980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1527f24569b74fa21dd494d8a66c7ad8bd198a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312025.20/warc/128dc27d42904a7ea14a0263f053c4aa03ecc2f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312025.20/warc/47b07e4a8e12da3b1537c2b85f0cd73ac878a144.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312025.20/warc/772635b359081fba825dcca659ef910725ee0d6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312025.20/warc/a135afa3ca9b40c3d2496fc6f5cd8c4a854099a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312025.20/warc/b430e529d7948f806a05da016b8113e5c47e722a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312128.3/warc/6fe46a1a118acccbfaec56c0f0d05f8cef7fcd60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312128.3/warc/8086b4de2f1d179e1148676a3bf2c8d0cec1f7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312128.3/warc/84396525621624a4ff4ea2dc512045631eee308d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312128.3/warc/b47799dbafaf43d0e6f25ab33705f7711e4dead7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312128.3/warc/f872549dddae6125bf91438082e7bc0d66c366f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313259.30/warc/3336394835f54d5cd1f353d3e6f87c16eff2a6ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313259.30/warc/dba655908ce2bc8f09e00b5e53a46a1e499ca691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313259.30/warc/f32ac2fe00637c3153184914362aa5eb0237d7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313259.30/warc/f5d109925a780aa0751c6984ca349d9230917b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313428.28/warc/292ad5961e43f4f17545616cf2123a1435c9bce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313428.28/warc/3d23d0711591b7ca89e6cd82578dbd25f874a3c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313428.28/warc/59f3884220fac95c71b0814f6e21d87418fb0004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313428.28/warc/8870f231bb2dcc80eb0620626c4fddb0a53e41dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313428.28/warc/d13011fcbc05f1497caf2701c105358c45edc6a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313436.2/warc/106dd3a1c4ae4e407b993040f76bbb8adf22b92f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313436.2/warc/3e0ac75d54f5c450286c53a8fc222fd16a80568c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313436.2/warc/9950f5c4ed4dc8c9c0eeac85c103e38b43783cfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313436.2/warc/dcfd37f34ca8aaa9a03d9901d734c4543299c6fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313436.2/warc/f286cc8337a8febba4ed1ae7a11874a116b83de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313501.0/warc/8d611d3ea7a9a500c1c374c319cb8e624e7185cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313501.0/warc/c2410d00227432f971df9737d8edec98e7ab7f11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313501.0/warc/e587a4dcc9ff1634bda661710d75dd567589dee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313501.0/warc/f8b006c396454e7c15d3c6fa07c2741338c1e2cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313501.0/warc/f9ef686f880f11ee181f84b17110462d62e545ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313536.31/warc/32e28d2f658933e08dd589ac12d1cef39940db98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313536.31/warc/49cec5f8aeb46d3ac2586df9cf90fc26e3c4deb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313536.31/warc/586a0dc1d4b66f9cb5ca2525072c5d03450469c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313536.31/warc/61b9b6b5971b5125374b53f97863de54c0937b96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313536.31/warc/6b5b276d0eeef761fbdb6301a08aff54df3691b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313589.19/warc/36613ea2cb7d0c1f65c0e5c4e972394b2b586985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313589.19/warc/6d04bf02a035be6aae1885841188399e47be6a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313589.19/warc/917e7e7374bc381b999267152892886c9dff7d75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313589.19/warc/b63f39ca87e5b43a5b255a8eb1ef1f136a626224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313617.6/warc/2121c07c93cce341febe015f6910be6ea499a07d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313617.6/warc/42b07032c5d8c063147b564dc4ad33317439afc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313617.6/warc/5fbc0c6d738119d6bb39caca1d10d448612da1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313617.6/warc/af24135ae0f0806f751175c3bdecdb96c5b2075c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313617.6/warc/dc2713fb71d081a40430b8f4e6a16a1b2f4cc953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313715.51/warc/3c8de09070982edf289a226d575e3357d2cc0094.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313715.51/warc/7e9f7b7f6fcb7e9dd423d46e85393c257b707d30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313715.51/warc/92a32fb0ce131967416d6763d2b8dfbe457f03d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313715.51/warc/9d95be1de3985a024e6c5db7b97c46cbb5895a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313715.51/warc/affecdc92f7ab2bb50268ba24a2233427f82a3ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313747.38/warc/4640a577a63faa2f91f838f2031751772f8be68b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313747.38/warc/64700fd4c4680f7932b17240c067f296fef18243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313747.38/warc/af30ee3773436d709d61dcdae8ff15a77ffd5b58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313747.38/warc/d0a1c89b33fe36847a10c20f73979d7928d80a86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313747.38/warc/dcec005507fcd7932d0ac9ce41f1d147a2d1fd0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313803.9/warc/13630c7d743a6f1c27cfbd37888e314b5774fb69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313803.9/warc/bdd013fe204d52973f23ed456930e015fde2d521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313803.9/warc/c7c112744db431dcef4aa412fbd9f4c4b965f6d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313803.9/warc/d68004e8790b6e035b141b2fde61988bfa5bcf19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313803.9/warc/d7bf37498c149524507d231a48279db420c9698f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313889.29/warc/7973743270eb90026286ea90fa04346a977718ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313889.29/warc/a204886543c56bf92cb48c8b10eeb5224c3998ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313889.29/warc/bc8d287ddb1cb032b304464c33bcca78ea27d867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313889.29/warc/fd3f23b86a48e0aa1d54258b45b243853cca63ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313936.42/warc/1975bf1979aab11f68205d9c37f2d03074ebc5c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313936.42/warc/7cc2e6d2edefca3d5067db8583ce9f02c1064b90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313936.42/warc/9bad16b2fe564ef838814fa91a7d08ce276aa2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313936.42/warc/e2154eb97db667dacfc0f170a08f4c19e3e1c88d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313936.42/warc/e82fce01b6f4d5841142ef782e1899958b33b39c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313987.32/warc/08cdf99690d2df5ecad2421823013489528e5a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313987.32/warc/0e5a41a98dba3bb5167a5941591b1a886e885cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313987.32/warc/5665ccf592e1c70b3750d9ed3aeb01a6f60c92d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313987.32/warc/e326fdfe74560712bababf1574a5fbf67eb727e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313987.32/warc/e6f241175470b019cb437887cd2d5dd47d3f1bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313996.39/warc/8ba2e13979ef07df62b9927b34804faee2b88cad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313996.39/warc/a60cd15538d0f467033adafd1f817cdc8bf4cd60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313996.39/warc/ab4652ca8322fe0624065d6cd05bea7a7f325d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313996.39/warc/c259a185e2857190b1149c52aa8aa334ce3c840f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313996.39/warc/e34407df989700ddcfd4ebafda24bf83e3d6d2c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314130.7/warc/409b06e7a5a3dcc9c3b8d4395973a29527ea4289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314130.7/warc/40b37b53a79ca14795098d00450081fbbce68a73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314130.7/warc/a947cc33ff6d91397e98e85e272813a6e849cf71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314130.7/warc/d9c2a00c7eeca504a8eae8d554bdd4c038389646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314130.7/warc/e4c83da711d2012d26f14ddb2eb105b796cfe246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314353.10/warc/3c8065c4e0579745e61fefd47681af1807f055c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314353.10/warc/80d7f231c5cc6c1731b66897f9bf251ecb35dc5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314353.10/warc/c66bf390854ddbc0d3b594f88c583caa736bb485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314353.10/warc/c999e69e141bfc72e7ec73291ba35f49358918c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314638.49/warc/23db96401b9923d38ebde66a7bd75109dddd3e0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314638.49/warc/294018bfc73b140a69e9aec53e52a743e352065d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314638.49/warc/84a7e42836a382d287769dd341b72c61b0e4cf9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314638.49/warc/c8aee7cca70e2c7dacb228996bc8e697254caf35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314638.49/warc/cea69b6846517c613fe60060561bcd16cff72123.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314641.41/warc/48379469f4e53a45c40e6dc5c2844e4f7b68a522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314641.41/warc/7cf83cd8685e04a9de9abc635d28c680816e61fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314641.41/warc/a412f4933a70d2e6856e53656ab4d012bd2ebc87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314641.41/warc/b6deb822ba2d73abd02f549c31cddda3015c9e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314641.41/warc/f818f028e0d0cbf2fee5e70947d1fda3017624c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314667.60/warc/08a1db9a7a0939367128e6e6e2b81804c66b6018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314667.60/warc/897bf0c103083bc117df662dd3c1d76bb6eff497.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314667.60/warc/93f712f9abbd60acbbbe854910b2c75ad666704c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314667.60/warc/e814ab889393ab69c5c0511cc17424474821a2e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314667.60/warc/e92d0ac3f4a42c7711740edf56f445250f6cc174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314696.33/warc/11c741616487685251dddd3713102eab79bfe679.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314696.33/warc/2849186d3a79fd63cedfc1b9d98ea7d94d126b21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314696.33/warc/6ce40cdb7d37931b6d017ced7b62f17b924cd674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314696.33/warc/c691951336e6119dfb9feccf02526d404e47e373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314696.33/warc/db8e47fcc55247babb2426d2f5a7594353a82e4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314721.74/warc/0ad4ee4a8702e4f5bb6d652c585ab5743961419d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314721.74/warc/0db5c5c6892e5286624479e6b96c09291f3f227c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314721.74/warc/474b9a032f6a78f6f911f555c30ada52a5cd6b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314721.74/warc/8d95bba858c4309d1e4eaa9dfdb992a94210ab3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314732.59/warc/7755b538a90d8741daaeddb925875ab864a9b344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314732.59/warc/bb1261809673f905e205654ea429b16220b38740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314732.59/warc/ea352764f0d66bb49b0465416c4e5427efdc5bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314732.59/warc/ec126825331fd77d2f1f49f5c7761eafa0072f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314732.59/warc/fdbdb813123ecf7762ab2511e2d19b1ae767ee74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314752.21/warc/4434d472cabaf0df8b688a9de385ff1122394491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314752.21/warc/54e9175982db4f37fe96b343e485a04362c0e1ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314752.21/warc/94d9d97af73ae9cce7e54e9ca66b65e7d9c02d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314752.21/warc/a64de1328d5a74398b3185a074986d6d65f4cbfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314752.21/warc/e8a81242ca236ea217f8bbec092e26570e963dcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314852.37/warc/1fa9aea16b861a1e52cb6ded4e2dbb95bf737133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314852.37/warc/4d98aa45b7eba3552123bcf07c688ee6d3e1b647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314852.37/warc/5d05b709c25178cf79df0187f2e38c2b03bb6ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314852.37/warc/9c136806cfcd1116b5621388b82da78dc5007bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314852.37/warc/bab02b5e42255c1f166a5ff3e3a69817de70f9bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314904.26/warc/39622fcc3fa3643694599dcffb6436656ee73703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314904.26/warc/ab90c1426ee605adfb1fb40cbb44ecb4291f062d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314904.26/warc/ad3dfb6a3cbf7f25e0fa8bf53b11213a35cd61dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314904.26/warc/d360328b63d60b667c8a52fc1b1faaa6f78678bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314904.26/warc/f93a54e478342b9914522a9229dd3a212219fbc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314959.58/warc/4b2eb859b1e3835ea3ce78533c36bf944d4d86f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314959.58/warc/9ab4a11c7b586999c609f589eb5f91bd013e05b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314959.58/warc/dc0ba603be082ce3d07fefd33038ae59bc736125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314959.58/warc/e24ad16ca55cefac30101d39116248c814b2a75f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315132.71/warc/04457c4eb7fcc9b7beaa93764791e0ecec48471c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315132.71/warc/3b401a8e5c613ec27ddcf768cc748c95b114bf31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315132.71/warc/8bda8ddde1b9cbd62bf6cc15f51e1540a907aea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315132.71/warc/958b847dde8ba234ee253d6d643e52b371acfad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315132.71/warc/bfd916d8d4060e5d1c40c7fe9337c13ba88886de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315174.57/warc/3adcf1eac8ba991bd33931b2d8074779cb0bd5eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315174.57/warc/457feaee971fa3e64c94175bcd1f6a5a3c985caa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315174.57/warc/91dc3360b34cef9799412748cee2ddabcca9c86c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315174.57/warc/a34076c1958c42de33339baf1c0b60b020c3a141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315174.57/warc/cd61d63c1ae4438fbb86c02072297d591340b388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.14/warc/0698cacc325d925464af37f95ae2426e2a95ec0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.14/warc/371819f08e6129b18a66278ea545f37199c66dce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.14/warc/71ba1e03d4531fcd4dad84b098eeef4e61e28326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.14/warc/95c4004f3636f4657fc549c7c12a72c58f5e8d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.14/warc/a681d1c6425bb7d53ecd2f4d0fceccf200f59b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.56/warc/27568ea0acc00e5763fbbf2fadffcefccc89b201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.56/warc/6570478d5e8e079110bd2683eafbb73c09666fc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.56/warc/8aa1781b5669756f90ddc683de1c07f8fc35e2cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.56/warc/ae225e902c6658b731c44422564a475f4dbc3431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.56/warc/e873263e3e0c22773505465b99f7219991b35472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315258.34/warc/0988eea3117198fadf3ee51f1025c76660bb2f01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315258.34/warc/4735f4d5e86f8eb8bb7d71594ef5b44e1af584b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315258.34/warc/7503959b87196d0565c2ea4358fda66768280066.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315258.34/warc/78ccd513c47e6ee75aa207c2e8aa58c57da06d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315321.52/warc/1ed1c62c22743ab85898f9a6d159c6e7dbd26357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315321.52/warc/38f29ca8be31dc1575101599af51fdae58aed378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315321.52/warc/4033f64f46587315be13a08af04acf8b3e289419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315321.52/warc/4bb5239bf0e11083c41b082be58cf824fa51f39b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315321.52/warc/601a5c8b4fcd59972ce2844269e6efa171656261.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315329.55/warc/2885902d84e9afbaf85694d4d21e5652792c2406.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315329.55/warc/6409d854b4216417efa258fd8376102651024176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315329.55/warc/afcef1190ef8208db3741ffff9c162b03d396c17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315329.55/warc/e39e044cf57e0281a0b254dc99ef420c90c96a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315329.55/warc/f8fc9260002a5e29991f0ed31c668c3a3c5c61e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315544.11/warc/049f31c2eabee5577bb33abb81850afc21d82aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315544.11/warc/5116a5db356cf7da9670e0167340e22b08d3d016.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315544.11/warc/75f680485e88a8a599abfa4bc9c862554e362800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315544.11/warc/78dd0718db358eab27cdd37b0e8be60ecd744740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315544.11/warc/7c3fbf598243c0286838ab02b2c5d83878349a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315551.61/warc/0370dca89bef365b8bd7afb133a9c0c34e449472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315551.61/warc/17fee55509f89971e2f7e82ff16f76a63e649a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315551.61/warc/3f5260641dd5745e3a55da5cf90a42708a4f861a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315551.61/warc/6a2ed302ade3c275c2f7e4f90f4a7f9281727705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315551.61/warc/f8e00ea5824fbb052f01a554afb4b921efecbd55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315558.25/warc/3a9d4e93e03ab60ee304c42f296e9cf38f9ba746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315558.25/warc/488b492ff6326ac6db924715d4196fb2b07e5318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315558.25/warc/a0c83c142da022e91804e2245f7174a8c531d265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315558.25/warc/beb7dad337a2a79b6328cda8b7347b12ad69e590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315618.73/warc/53ea3fe82de88bb92377281c12c2bb84ab5e15e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315618.73/warc/75b26d959101ad193d907fd34a729dbfdf1b0666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315618.73/warc/b1bc80293e4141203a8f28f93894590447b8ecb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315618.73/warc/b842c32d79ca37ab5674d07c3d580212bd0efd4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315618.73/warc/e0d269fc63bc9549bcdf04e939b3db3863e7996e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315681.63/warc/26b5e08b759c9c4b606b4f3de9b9907dbd48aa30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315681.63/warc/a4709b28d37735325cac8a8332b6e4f8f9c8c099.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315681.63/warc/b6093dc0f3ddebc57061cbf4a680018282c0167b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315681.63/warc/e0a3b71c78df410e8ffc83f66cc737c3fee59281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315681.63/warc/f0457c73f2abdd9e8e66a5a2ba7693e188965f8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315695.36/warc/34c06563b580a1c9c565dde6badeb2beb92db1cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315695.36/warc/8209b2424e4dbd43dcb044a414e5f5d8eca7cab9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315695.36/warc/86fa7e7f6f8f0429b4ea2259232727f4da293d50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315695.36/warc/99c0e1a5da1908d544c462b295247bc4e59da7b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315695.36/warc/ae2d297e6619e01267121d8f7f651b8ce11d3f20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315750.62/warc/50108c02841086b811fe055cfad9529d5fd91c6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315750.62/warc/673dd2792d4fb8bc12070b7a009c6b6e74f0a20c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315750.62/warc/8bdc5011a4d520dbd699b660da6fdc809a6e0a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315750.62/warc/c76d10164b2bd7c0ce9ddce5fb26b44888f4f5e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315750.62/warc/d3d1307159ed6edfaf16493c559b69621697b3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315809.69/warc/4e0e5db80d39203c8436884402b963203f6c5dbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315809.69/warc/6f53232f190a18d1e7b8623fefdcc9ff6161453e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315809.69/warc/c7e570a6100504ba687e76bcc06416b56483b544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315809.69/warc/dcf28054cff055dddc97c645c0f5afe61a59ae1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315811.47/warc/43b6348335504beb888579398f17b24df35f6aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315811.47/warc/5f6ff7f349fe34240f2de695e5102be8e16a4cca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315811.47/warc/6d6868dbf3acf02b5dd820f09c3ffd16f4e8b009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315811.47/warc/7411193c3e8c30af1bb00ecc30dfcaedc383a8ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315811.47/warc/90b0dcd993b18cc8cc819dc2e07c93829644774c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315865.44/warc/1c83d87fe2d81531c5ee96ae374d72c894c00a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315865.44/warc/802e0eb2fde204896550b81d9e0b5ff1a87a205e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315865.44/warc/c35c951f6ef7ae20f1621aa66b9cb16bcd66963e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315865.44/warc/c56a1a1967a9fcbcd83ac441a4232a8f6c990570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315865.44/warc/f6c9eb3d88cafbbed5a8846929a32fd55d1368df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315936.22/warc/04a17b6d9611382bb42e0a8f62894d565068297b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315936.22/warc/0515a0629bda9f6d3ac08410753ca9b672ac3959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315936.22/warc/524674f654b398dec3cd147017a9fb3d98390cd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315936.22/warc/b08d88de2decfc0a70d79958d595e7eac5e4696f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315936.22/warc/ed8006a1508d1fb8a474ebb9d48e2f9f0626c325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316021.66/warc/246f863f75b77e878441f732e857a6acfdc0e736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316021.66/warc/39aca1b299dc0c52fec84a56741445fba2ce8d79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316021.66/warc/aa5d984f03872e7824c86d295cd090d0166aa521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316021.66/warc/b8f84b57cdcfa59767d1040ac3a3832b1ffb3ec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316021.66/warc/f9abdc94750139c6c8bd9dd08577673bbe26db00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316075.15/warc/6de9620a6e08f52b6efdaa457c8ac3b6b2b4a164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316075.15/warc/9bc8d1a9156cfbb899ff51807e410378c3eee3d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316075.15/warc/b1ce4b09fbfa21f530851eff67c82dedeada7a34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316075.15/warc/b92ee0342c50987f9aa5a197ee9773064706f4c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316150.53/warc/04d6c03e307e945116b9fc32bbbbfcbfbb6e6320.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316150.53/warc/3e2cb1c3b221013e7d35d11af22f914a2eac368e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316150.53/warc/41e0621c6fb4c344b43e522ba235d5ae7a5848ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316150.53/warc/66b3c35af1802cc41445cbd7df5b06b377efab6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316150.53/warc/b219439414b2b5a84013aba31a822396ccd047a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316194.18/warc/086d8ddf376bfbf51a3aa25587cc968d033f977e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316194.18/warc/2b8a7f79f2d8cfa6940b1b61b3baa03c7ea89795.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316194.18/warc/3002737390be22c8ebf63fb71525d93fa39c0650.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316194.18/warc/5890a8b4c5c1e41024dcefe8d32ad77d8bbd684a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316194.18/warc/88b0b62ca777fdb44aa552512024c58c1b1b57ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316549.78/warc/1e822bad8df194e3ba39b1d21cea964fc6826e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316549.78/warc/251a06352793aace79c331d8ce75868f668baf09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316549.78/warc/5cbd4ff3eccdf1e5b4296db7be36bc9193147925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316549.78/warc/bd7ed6940a75f3a7c68c43337647215711fb34ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316549.78/warc/e99d1ef82fe3184545b926cef9d16ab6afe305e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316555.4/warc/4563075ea4e113a81cf8cc2018747272cf322e95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316555.4/warc/5d2d30e6e2a10784f65cdfcff9221657fdd2a606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316555.4/warc/680ea70d31111bcd16c06c510d0681962ef96637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316555.4/warc/6fbbbb9348081e4f39e147bd1c95b21f1dc4531f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316555.4/warc/faad202bb60e27105c0f29c71358b271ad5cd6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316718.64/warc/564634a1a8f0efc972f958cfcea621781c8fe9a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316718.64/warc/ced419594208d906766eff131b220fd205370149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316718.64/warc/e6e100ef3af0e82c39a9eecc151b1878a46c60a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316718.64/warc/f144dcf0b76188c0aa15d1d6a9f68a6b0505a1d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316783.70/warc/3146b2a987ddfe6ba805987faf543faa634ab2fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316783.70/warc/345e8140be1d477cc5d565b29285fc1cd3e2f5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316783.70/warc/8321cf39f612f5e6402ad6055629d410e71fee49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316783.70/warc/877bc92b29ca7483ad87cfcb0318c574dacc83a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316783.70/warc/eb4b150f2087c646f1d34835f33aca4a2f381790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316785.68/warc/01c372618c78488ca37fe081364a24960bd2c295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316785.68/warc/08e1cd1123782c6d0688b25dcac6ffaeb7e02e39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316785.68/warc/163ae47bf849ce86d89639b39d5386842e39dcd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316785.68/warc/1cc47e52183a5f32850916ffb941596cdb5b2e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316785.68/warc/64626da2a33e875ca611c6e1973e618fb1f02278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317037.24/warc/0d3585f31fe5b85fe5f3ea31b620e9f5a39ea336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317037.24/warc/6040712ce349621d425075d6cebcc3314cb03ca5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317037.24/warc/ae849c53635bee83aae4ea1326f27ddadf1f9a35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317037.24/warc/bcb35b28f28e5d0ea8587bd4c0d411585c37b849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317037.24/warc/de06d7243a2e1acbf4eee92443dba2317afc4d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317113.27/warc/27866278cf27cfea50a3906843c0ac787f823e03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317113.27/warc/37f3821a53cc538c141cee10190094a56da976c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317113.27/warc/6d06630ca63638be606ee222de9fe1cf29c6dbbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317113.27/warc/9cc65e6787587051f4dedcf0b1b8b26edbd4824e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317113.27/warc/c7e5afdbf374ce70230ecb4e91f767af56fbaa08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317130.77/warc/4dc1632432fb1210aebf2299578147947de59f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317130.77/warc/55e7bb656d0274fa8910929a3a4d3b957b71ccbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317130.77/warc/f5e5e93ad45a126cf7b77ed514643774c29e3e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317130.77/warc/fb861c66163c919926d97655c251acf6afbe4f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317274.5/warc/0e1ea3cf823342d89b844f80af8f879760da1cde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317274.5/warc/2badcf769a85a1d55bebe47e596ffff91c72b89d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317274.5/warc/635dc2e973cdc290fcd48c1739d2b9584067dc96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317274.5/warc/6dc3e31067b540142e40e50566246667c7fe194d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317274.5/warc/741b380019dd0bfe1c4f9a28b00104b51da1df1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317339.12/warc/27c6329e0675437b2f8ce2236b717a2586f5a860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317339.12/warc/4fd01c2dd67a42ce3822df5a7c9b7f7d09d5ea86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317339.12/warc/91516c442d818f8e09d2d6a3bdba0259b3dd11fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317339.12/warc/bda71a2c3593c95684addf13d01dcce0be6adaf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317339.12/warc/d1204f3d34906f1576ac346c7c578b411845532b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317359.75/warc/2911d13c94f76a886cc0b0de2d5e7cd6f8d3c20e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317359.75/warc/4fb5672b2fa0407ef675594766a0b33b8d5189b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317359.75/warc/8833b63832e4e1720db1b75aa8b47aff58920266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317359.75/warc/d39b1908f17fd7ce9c68e26ee6dcbaf2dee58ed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317359.75/warc/ded3a9495a74c1e7682f2bca2de9d99e9b6bfb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317516.88/warc/96943f4b1b2d51d6dd5cbff2099d62fc7bd62729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317516.88/warc/af92655c7052537c7bce92f1c00629a1dcd2c4df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317516.88/warc/c9b17fd4ea3990f2ae2ab54dc2cb7d4b44d221f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317516.88/warc/d31a6397ab05b454ef91bed0c581d716fd69bca2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317516.88/warc/dc391d8e90f03f110f2fd9bf81dc2c76f5c8c492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317688.48/warc/3617748ec735d94f7bab613bd72d59a72d771edf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317688.48/warc/5cfcbdacdf1d98959c257102ccc29cf7551e523a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317688.48/warc/6fea2aacd77e8da998aa55ac7407a36187525a61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317688.48/warc/f90227741c9881b5982ab54fb939e7173f60cba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317817.76/warc/2c6ab8c70a03af31bf5ed6353edf4d7fa68fa2ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317817.76/warc/352ff98291a05671aaa28914c22c406c16861e2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317817.76/warc/641018262ef040198ce76f3de0e275a3e263e5e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317817.76/warc/741068685650931bde984522b90a40fba11629d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317817.76/warc/e61a555fb7074063aff4b9935526a84180e8a331.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317847.79/warc/05b661f98d5da23fe96fb7acd50b1e40ce387ad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317847.79/warc/3eea89d5b6af25e7ba58043c3ad4ebb4b9268ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317847.79/warc/4b869a16557196d38322162d238c3b4d59f46056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317847.79/warc/a3ea739a7457c6f05757c7d40c080594e2ac3ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317847.79/warc/f1a4e41f1fb69c814a544c9550bba5c9fabdab8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318011.89/warc/3af8a4f60fd120d28469005ca542f7522eddd23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318011.89/warc/74fc0eb5fce37b1fc24ce6eebea9d5b462a34ff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318011.89/warc/83be98b910120bcc15ae31cadfb6d868750b7a8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318011.89/warc/9ac0e1dddb49686487943d17bb35ce899048e8fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318011.89/warc/c80f0e749da0fa3a13a0354b3a8f3b8690d7d46b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318243.40/warc/0ff1d426a0ba8ffb37c55f152ba38dc5ccbbad1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318243.40/warc/3a75665fcba5550637ce6525724eae3c27936a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318243.40/warc/48c8f36b7244c6ce659142173c41c0ebe738a7f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318243.40/warc/e4b9c82873a0d22566eb7b4d71da9253be2587e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318243.40/warc/f9f4a92bd444d244b60fbecf91b3731cfbfd12d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318375.80/warc/2d81b1d78588f657906aeedfaebebe1ebbfb48ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318375.80/warc/8e6e838a748ec196dd2d0537944b04b9c3ef9dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318375.80/warc/c2db048aa011f547947f9ad2c6e0f1654ee831ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318375.80/warc/fb291315437647cf49af9366cbbb0b3f0098344b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318421.65/warc/0e31cb53738b0ac55b2b3605e7a2d75bf46150f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318421.65/warc/370268e66cd2d89408d127ada60b918d059ffeff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318421.65/warc/495dd6b422b29b934642f5fd580307e728744528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318421.65/warc/53880ad93fbdd65bfb361dbb78a4756c0c7e1baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318421.65/warc/b854a2d22fcbdad520577d3ee9a6adb8d82ef64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318894.83/warc/131d9894ed1e4d71381b361266ac79c4a621936d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318894.83/warc/35b388387f8bd38fc847167ea6cc9021eef1423a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318894.83/warc/381e8aeb2a27eb8792c2d575db414bc057d05004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318894.83/warc/bbce6fb5312001cb270bc1bbc60e19eb2e939421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318894.83/warc/c50aced1961b43e7b26e446775572b2a586ce88c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318952.90/warc/049aaf2ff5eb50a7868b7ec97ba0c90eeb82b84d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318952.90/warc/06ffcd898ba504a4569ae065e3eca60cbfec2abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318952.90/warc/0a7d9f8fc52259a96f715c5b5a1cc8f520fecb4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318952.90/warc/9151ba0fc81a1f9ce741d75203bc8081579792a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318952.90/warc/c2156c7b78ad59d0f3c2e85ac83dbb46351a2fca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318986.84/warc/1c4b4be7d740d51527abf90f5ad856ce65b064fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318986.84/warc/1cbdba815c5e3184d7d00e1fd56535cd133d584b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318986.84/warc/3082d4347322ad56a8715ee8df5e538179f083b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318986.84/warc/99a891cddd18d55f550260c15148fab50c878e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318986.84/warc/ce1cf6cc5ce3d0a353142ac44739c42361372c22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319082.81/warc/7952157a686d4d0caf8341b734e778cc0108a3c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319082.81/warc/9f02252eb08662860dda6e71c2084fdc713a3c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319082.81/warc/a522d4469a21c23586020331d762ff60bf3e0c23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319082.81/warc/edca4e620b15940517b6945da92f81a7708349ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319155.91/warc/102366991d39133014fb4d25b2d1933cef7f3da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319155.91/warc/61e4f3b4bd82efc4bfaa2d7a70dd1c508fd91a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319155.91/warc/97c657e95659439dc58e05e9890fb5afab2d0619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319155.91/warc/f4dbe8d85487c21cb351d839db3c4c47188b3e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319155.91/warc/fab12128be1144bf34b08f30b550b3e14dd2a601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319470.94/warc/6e923f3efd804f987ee298ad3213280f2a103bad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319470.94/warc/85316aacd6244b87f34dce97246891f5ea524736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319470.94/warc/ae9fc2d06add91609910f5fad00ab3adaa2ed829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319470.94/warc/ccd8fc5a2fa604425fdad36b6abbfc1fd3dd6bc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319470.94/warc/db6e1e58022e62f0573bb5063fcb7b42dbb79534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319724.97/warc/18088209958034fcf342bf6d0ebab5fd2478ee8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319724.97/warc/4da75cecb1b9897b5283ad8c0a334ee0f86e4d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319724.97/warc/8e373d34e1d32d66cc0dffa8df41864d9cfc556d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319724.97/warc/90b145eeb9d9354774041bf4920233991e68db4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319724.97/warc/fcfe2c5058a021d3a0f7bccee7efb15462cd64c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319915.98/warc/1bdcd003a08a0dc21e510074d18f6db8f595f8f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319915.98/warc/20321035e56bfea399f53c25adb34319df810b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319915.98/warc/66a9d7a5542006917a0ad866e474083c824d9aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319915.98/warc/7fb1ee6c39ebea71e76d6e5a72bc1658f93af177.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319915.98/warc/d8f2d177b16e13ce3630154c5c68a948b0336779.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320156.86/warc/2c41746447a7af69cf8460746b8991b0d16ff53a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320156.86/warc/4d6c0d3b81adddcd405538acc06a67395b9452d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320156.86/warc/84bd42cbf94a62332f09ca3d33b77cc2bd36c08e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320156.86/warc/f278d5680e5d0a10b2a316e03729c6c2fa0c61cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320734.85/warc/2e0a318a27bc5a5daf51d4c7bc7add5f592bab21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320734.85/warc/3c9291988b2df515c244c4bc764bb70a4afe98a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320734.85/warc/992d643f54e5a2cca8041ea856fb1afa381c0db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320734.85/warc/9db5dc966af34b215f7e46847c3b2c67018499b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320734.85/warc/a818b8886b33220e3bfce79d3cdd08a70e7069be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321140.82/warc/499f91ef6fcb637258dac75587c0831028430452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321140.82/warc/a4e98db3f28edb3e1baa3358e4dff5bbb230d190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321140.82/warc/d41ec9641b3e740c02e3e11220db2d42bd1449a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321140.82/warc/f849bd7aa63f151ca5f61437397364531e31d401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321140.82/warc/ff1cccaf8249d65e1cb3dec4a26324b578e3bff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321160.93/warc/4ed88ca095819ca4c4c9b121aa7eaf5e099e9db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321160.93/warc/94a52d667770c8fa2a20b2006dcbac1bbe6fb914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321160.93/warc/991d7c4b0d9f3198dec94fe7ec41fc5e28cdcb3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321160.93/warc/df74761fc9d4920a33f288936be2cf5900ae1c37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321160.93/warc/ffbb90c613c49811f024e18dc74d989c6d00542a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321351.87/warc/1d04af0ae772984f0856ea5fe9fd48e074133aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321351.87/warc/3edf3cd4714f0102e6fb54e33d6c56e30029c7f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321351.87/warc/7aac1a27bd4c5fdca67db0cd38737805f5974400.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321351.87/warc/8bb6f478fb15f9266115a8caf61c3e7219c9e444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321351.87/warc/979276c1b9df7b6d28b51fd7ffb7af4423d159c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321696.96/warc/13174f655104fecd7878980618b359a90f6b3044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321696.96/warc/30b94a9b1d750deb9f4ed65a253e226186efa881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321696.96/warc/3a3b9d0ccbd933fb6e0e2dcc4ad59796da883ea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321696.96/warc/785f6af2f3207e6ca2c564bbc4372f7f1c888312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321786.95/warc/940ee35df88d82b8129a5f10f12d516a0825da84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321786.95/warc/d8e5c89573ff2416c088f32c475ddc022418670f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321786.95/warc/edb5891ad1dc8855cbaf39ad9bc509f9a40b610f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321786.95/warc/f5fe5b99a0a16bca742bbf7a10f2a258de9fe433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321786.95/warc/f622902bc09c2f3f80d0e227509f0302e302a6fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322160.92/warc/085d7a0dd3d528a9be2f152752f909cdf63252da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322160.92/warc/9d94e1527609c79516c62fa5100ae83b32c30707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322160.92/warc/f4b3def8b67377441d277d4aec2496f9668b3e1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322160.92/warc/f5d2264e83dd7e4e02f599813a6a851d5ea18188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322160.92/warc/f8f0f49ddfdead94d468f39cb9a0302436d7ca2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322170.99/warc/2bd3d74be630053f0f5554cbee9cc5d3bdbd93a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322170.99/warc/2be7d9b7be3ca9c2fff039772c4265df6df7699b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322170.99/warc/5f435ecd6f55dc6b6b3fe49f63149caa96e6e02c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322170.99/warc/8eaa1b9d3a05fc103a582b42898874461ddd6da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322170.99/warc/e6fd6324a30fa4a08154c4b5ecf24394ac9418e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323067.50/warc/1811c84c1be7d97cefe56e2299e777087937c0b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323067.50/warc/4d001b1ef26421123ff6258eab4a4c60662ae6c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323067.50/warc/855c31d9a7e31535710632b938ced46f2d0d76d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323067.50/warc/9e08b71be4e48843f6c53531359b0067bbaf2f7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323067.50/warc/fa6ee2ecc0576bae13b86d7a3112c8e466d07efd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323221.23/warc/142f186e2bcbe5d4029ba5aaee6361ffa9de8c32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323221.23/warc/1bed345097bccbb538ce19753e1a22424a9e9ec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323221.23/warc/86019dc53ce3cde91d225622b174171958704860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323221.23/warc/f56150415a4029ba5d3c1a7493de21b858f9d1b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323246.35/warc/027537772b8de30c9f4b70e0438e21299d8a1512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323246.35/warc/1fdacca90ca18c57cb471d417c72907940851cb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323246.35/warc/215d8c3ebb66b99742840d284af900f0ad03a9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323246.35/warc/9be51f9fd83f2e922396bf9d46885c45c3f42c07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323246.35/warc/ed02d959a131bc2feddbb1556e3c31b14be30f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323328.16/warc/703de4c8a86e3035cb8596099b453447eb05e5ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323328.16/warc/bccc5581915056d1d46a428b62eab34b44e85994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323328.16/warc/bd69b2ecaf5ca6ecd48988febef27461748aaee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323328.16/warc/cfc6c37fe690f28ae8dcb567f469e4ca2831411a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323328.16/warc/ecda883c54b5376b53205e2f37d947d44454b4a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330233.1/warc/1c3dd7590c77bb60b769d4c3d796f00c8b670fa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330233.1/warc/37486216c26613e8db72e2b3f68500a73d747f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330233.1/warc/835c8b3446a1f4474792b02ea886c9b8550866da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330233.1/warc/dd662967662e5db524f37109c8d8404190cd060e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330233.1/warc/f10125b59f140eebe680d994d47178846d81bd4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330750.45/warc/0cfc1bb8e8dac6a4deb67d38cc6f20eeb5464e0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330750.45/warc/3fff2c3e051fb99b2a20aef51303c39a4b0e5b66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330750.45/warc/b672bfc8060add303de4f027860c8adcf8117f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330750.45/warc/ba4643bdf4392e03b9863f794af76ba8506f8bdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330750.45/warc/cddf188e1f5f3d1fa8be4166f3393cdab3756080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330786.8/warc/1f8f7b3a4fad0d1c8b8ea31597ff93dccacd4397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330786.8/warc/b17cca2d931855f7ddbe8f7e3695bec353db00a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330786.8/warc/b39ad8c8627bf911212ddc216c9cfb0c00aff272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330786.8/warc/d802806a6a21e9a1290040f93a9d1bf23f76012f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330800.17/warc/20af27e75be66e13e48319da4c801c08b3bd028b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330800.17/warc/2fd572ab48034622a36f0da7a6a1c65665b67992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330800.17/warc/5a86c3e6c8c60abd44759327774cce8e8cfa1d95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330800.17/warc/e82d7cddc972123b2289e14a7016d919e9ca35ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330800.17/warc/f31b41d12b3be8bc6ddc3b0e42c34d9f519f355c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330907.46/warc/3c5ae2690bec3551012c14fbfb0766191eaeb211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330907.46/warc/6ca2b4d724fae0049025639c352d8dd3bc83ead2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330907.46/warc/911b8c740a741c6b0772124e5477177d92b95e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330907.46/warc/915a616311996d064a66f209588a9e0db2d096a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330907.46/warc/a0cb6d1c5c4b0a818eef52efd93f98a8f4aec2b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330913.72/warc/2431e888e72ed35ac8dbf8417b591e99b865628f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330913.72/warc/320249a11b38d23d0767f74124e9874563fa1267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330913.72/warc/35358c0d8a0758b0d6996e653c267adc3e265ec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330913.72/warc/4eb2f3e558cf7e808b64bc218af91ba78742b3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330913.72/warc/60e67733dded1b0683f03270204f001f16b8a2e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330962.67/warc/02c3d649f7f2f66dab997f44dbbc30f2fb34b0de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330962.67/warc/22fbdd5866738ae507de092e4ea6a41c7cdf98a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330962.67/warc/29d7e7596d4ede0423dc8120113c2d1a18502e67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330962.67/warc/7f37ce85d3407712452be3e1f2b37f95d4dc34cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330962.67/warc/8031d6c37d2e941f70120d1e8ddfdb70b2790ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330968.54/warc/514f2a4b7e21e73197c8193b78ff9504ec12c086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330968.54/warc/74ae7b031a673460aeb1682390b52a312095050b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330968.54/warc/82e254569ae07f83df4e977c2a1acd9811a55a91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330968.54/warc/a4813198fe39eca1440ab75a7ff6633b840d85c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331228.13/warc/16882341a5101bdb7a451952649a647a54b6c993.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331228.13/warc/29bfd21e96b8351408702f543db49b59c2760661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331228.13/warc/60c7f1b545bd75f7141e0a10af1d82973cb3ef40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331228.13/warc/d804d3563ddd423f0b11610a24d392babe811e0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331228.13/warc/e57110b97a367d025e06a8549f739123555d5cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331485.43/warc/0acd63472662c5190b673669133988bb6dc6a617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331485.43/warc/125fdf3697278b2ba506bbc6f46f65af42056f4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331485.43/warc/31986d5db9db738e6473821c804343917d5d0413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331485.43/warc/791e0d2634a90a04b2a4cf4750b79e8a7984fc84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331485.43/warc/de165864fb4fec9b7c9352dd096b299bdf7b5932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/15b08b419b9e17e384d534013978af4249422a20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/17187770333c668f5791bb05d6dbe0f9abf53bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/21fc09d4ea33d658fb635b0f788bc6dd7ca2093c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/24b4412a3b243f287a2dce9ba4531c680f645bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/25435c748832fa8a8df2fe324f838189355105ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/296c9e11d68c06724f27172a23067bdde4663a5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/29cf3855e5ddee41fe614d5dfd89689d13c50e34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/2ce2e82b2ba7ea1cc99a10548a81a7aaee572b90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/2f3bfc4c19e53f3ae1ce46778dde52f569daabde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/2f719419cc68d1515d2a1386430341656aaf398f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/2f81a655fe2c46b23e3dfc7d49583ca19f703e15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/3af68df8fc33fd71f2706f0b3223f38358386370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/3d12273d28f9248e2852fd735b4e0cff1a3ae7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/41a4daebcac4594ea077625a2fb25dd837ac2730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/4410b88bf5ea17568cb1ed35132687253d12aed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/497e6d5df0fda3b4ed2eb642057596c049dd7fcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/5115a8ab2a5882b55137bf9e5bc73e5f3d5cc537.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/518f5f96e92d61f4a1328ad213a4b9440af3c871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/55741fdb8a4acbe95c6581ae8b1dfc4a8be4ba18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/5d4d966efdbe6c36b68cab4d0f9439436b9a9573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/5dbb732a8556c48208ff2580ffe469bec8dd4878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/5e5431734f5755819924864de28f605ae45776e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/5ec6420bea9d54efba530ee1978038bfbfee5036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/61f6d49a57724b2229508d4d207a9abc007bd669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/689298a9869eb87f1037da9f418e3adb85dfbcda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/6e7de21eb6d090ec3f60b22976404af45ee69dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/72fa169bb72d35952571b008850ff9b00cae9276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/75afcc5b8d45abf80ba7b954386ad8d44cb80b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/775436aa0b9af0109fd4f9fd63768094509598d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/7845f102b59995f61f7acc51e79a2436aa2e4cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/7f52073220841bf0510bbccc1a15bb4f0134c0aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/82876a46f6b6581b8789ddc3b2f3584be267bdc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/87b91bb8f756eb2c3e7a8d7370999f9a7fb07f9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/89196716db104a278ed0280855443e903692862f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/8b61bce90e37ed146521b9d5e7ab6938a1acf28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/91065bfb707d1e364fd8fcd04db05aa9bbc8381e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/927a714f84345d0cd6cf045f2d26c4441a2debc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/96d7a213913a5ff886a7bc54df78e3ac86cdf168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/98122618c80d5dc7e89c732d8c4cb31d689eba6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/9a2c90a2096f29961dc7b6183729b581f0ca3bd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/9a50069e3c82bcd8fb60fd540e02f886b5e7a7f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/9f01996527bd642bcf2ffce255d6a05d358c0db5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/a23fe0f3fdbc2279e8669465e84a68c02b34e63b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/a3ab114b5d071b78b6cd649f1a26114002c62477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/a8552523c72bcfe21c39024bdf805fdaf1bda147.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/acc48b0faf53188b891e3ade4dd833622fd6765c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/accf60cb4e2bd8a0e35438bf3e46a1685f631ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/af0864f80f14f317b27f5639413d7fd1cd55d5df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/c0b9fb4f844f85e7e86f2347aa0308b188cbb4c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/c23b4da40bbe239923661a6a7b75878e28fd518b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/c41a69d9b3c789224c836abd90dbafad742b41db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/c743cf9b33fbbc6133b22af8057c92cc0ae20058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/c9bb8b839bd0474cce4dedaafc7ba5fff8d237c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/cb61e4467e700843a740e4b23df09ca178c906dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/cc8984e483bf87240ce603f65f0671b17dda7bd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d10b453df76477c999095711ec057413193a5f52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d39ef3f19a881ab7bf58ab99cd0236f72fdba40e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d718ba8083c5df4ed547f65c3ee638ca72e80aa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d758f93c1aceb4442b47a728eed0a170bccf5d85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d7a5bed24019963303dad65205b6a89b97711d75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d7c1f2f689012bf375d4e76378582b7e0a91bddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d96eae3390dc73d20380c0cb2259a4766e9c24fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/dcea1b22ba9d31e9ba7e01ec5ebcd950047289f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/dfadde20f40b638f97eea8a95019b0d0b283c15f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/e4ba5673f27b527e9e98b85b592b4010a00435ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/e6e5174aebcbe062e4f08a5070eac7e98ae4ddfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/f47a817d677ea3f0b344aad9cc7c6fc1ead993c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/f691b0cf47d812e76a66eb3cefb30b959d98f866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/f7b1b3e26fb52c01b3e15ce44709f9d53690a7bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/fc58a004f4eb4c370c40266f9593f012998694e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/fea93a274ddcd6a1f4cbc5d420e0f9750b3aeb97.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2019-35/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2019-35/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2019-35/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2019-39.yaml b/configs/cccc/dedupe-url/CC-MAIN-2019-39.yaml new file mode 100644 index 00000000..e710f81b --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2019-39.yaml @@ -0,0 +1,581 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/014f726a7e27777a552c3374a3b29f217f5655a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/0871195aa4c67dcbe23dc1f08981fe9404002181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/0aeac422bc310a9122417ece09960b29b714197f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/0dbd61cdf0a71af738f344c56aef725075709389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/13dcaa0de8c94c44218d6c5ee4ec347251e89f5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570740.10/warc/34a795f5c17efffc9750f48b3dd412479adbfbba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570740.10/warc/6a64e2b55299ef27595e357d05fd1d405bb2fb40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570740.10/warc/875fef2ab932ac72911f22c035343f5fa67d1031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570740.10/warc/8d0eeb0f82386f529f9af481eaf337b7c188e2de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570740.10/warc/c235e5bbae6170a973ca381a9311ac9f226627a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570830.42/warc/1d8d908593f80b5383ce231f390feadde47e6de0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570830.42/warc/1f1263ad90cba694a689c7b1b337dc70c6d6d457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570830.42/warc/715c99bfcd76540cce18c63576cf618ef673554c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570830.42/warc/74ce4141dd542e379c54f40d90c180fa149dfdcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570830.42/warc/990b792deefd53ddc26f56fcb770d83887a1a506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571027.62/warc/27fc402d6a9fc7f113565012f656dfc8b64cc0ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571027.62/warc/714fb813e6f6b350b58f9541cc270c5584cddc46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571027.62/warc/bb058d8a2f1bb91f0a4e1c9dbff54614927eaa81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571027.62/warc/da7e379b60f14e3c592fa111e537899a34ba4871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571360.41/warc/24ba2d74458273842b0df4a4971f9daf6af28af5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571360.41/warc/540b40df972b962b9170b5044bfcd047c47b8630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571360.41/warc/7fcbc3c53595900a7fb461cbf8cb34a1e76edb8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571360.41/warc/aa72bae0fdb692c3eaed55b03e98e1211db4b5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571360.41/warc/c4f568cfd3249126da72e8db2c93a46bbe188b4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571506.61/warc/20cd590b3430da20b0d1bdc36cbfb3bfc42b1364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571506.61/warc/27a0c91e82fa0b5896338b434a43ffb4c1aa4c83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571506.61/warc/666d1d2f8cfcb25ccdd29d021b1a1ea3a9ae811f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571506.61/warc/bd7ce26b7b6e7df8cff03d33c5e8915f832f4f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571506.61/warc/f04de30e53045c349207148fd606b38ad0588e18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571651.9/warc/1b30616c62c94a7a54a6d7724fbffe82922d4f02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571651.9/warc/2fd530510828b9ee395a75799851529378894d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571651.9/warc/3a87ee9e57f8c44726768bc8402648d71e0f6e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571651.9/warc/b3375ab8de13b7185215d2ee400d7e680bf4e839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571651.9/warc/fd1ed62bb34765d26155fb9a4c791df950acdca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572235.63/warc/186b0f8fa565ca707b20cf60b9e81f70e9476f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572235.63/warc/72e50f9a16e61c318b778c914d0e0993f9f7b551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572235.63/warc/9c6c5e6582d1608c7ff6156bb154c339bbc87e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572235.63/warc/b10cc4b3e1d5013d8269441e2eb8d811c064d7ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572235.63/warc/d755a9dcf85a98ecaaff691ec2100c3d50867f1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572289.5/warc/11bf943ebed1062b64f513f9652e9c563e644722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572289.5/warc/5a819f8d038b79693893a405abc4764a606e0000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572289.5/warc/6fc930cd8157ca413dfe00744db15fd5009927eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572289.5/warc/de9be143ffbaaccc5fc47cc3903aec9327e9e82c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572436.52/warc/2ca5b72f5bf39b69cabb413339bfba312981756c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572436.52/warc/4082b197f5fe012bf162a0cbc5b3db88d05c3c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572436.52/warc/4b8c8ec4e0d86d57cc31d9bf8d4bc86bcf94862f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572436.52/warc/d4936692ef76df46bceab11d1197c0fea2a0d3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572436.52/warc/eed9a9b4500e6fc77008e738b2ff84666b6b408c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572439.21/warc/11e359d9d7250fc7a3dc50d1cba0693ebe67e1fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572439.21/warc/25a01ca4f35945ec2349dd60bcc5632b0b41f0e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572439.21/warc/30149c261611a3b15295cd83b5b7263de112c324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572439.21/warc/5c69f08c68674331616e616bc4e54419e8710e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572439.21/warc/e6da87b66c7b264a0f9a909861ae76d6bc3bc0f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572471.35/warc/3c85d094bfa0241ebada6f0b491861429fd288a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572471.35/warc/67b257e6085134995c6228333827f7239d512dfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572471.35/warc/91d1e98b793648f360f75860659e1a74754f687a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572471.35/warc/ad256cc433cec62feafa3249eb6dc50be1d75646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572471.35/warc/f856e7d95e59882f34c25941242868546632e957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572484.20/warc/180cfb0fe708963da6e342817f65d5de2050c591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572484.20/warc/3cdc13279a7b5c1eb8d4184dcf7a20a32b361f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572484.20/warc/64167997fbc2eea2b529ee8820607d67ecb5465e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572484.20/warc/9fa97fc970c1f19e041211d84203dcdf8b1a97e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572484.20/warc/ba02141726b4f6659be97abef63c3f9165f22877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572491.38/warc/2075327269ba396f6a412bb5a650ae14d50b3298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572491.38/warc/3b5138ac111723aa4fb3e126fb9d61fa3775a265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572491.38/warc/7ee3f61f72e8ca59413393f4e491c740a3bd6ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572491.38/warc/f6fd26c4c4b2aa2c4b5432635c436d501514df47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572516.46/warc/3d46993a8a94d798603041e3f527483a8d085a73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572516.46/warc/72351e133751553989eac6d7bb318e96bee7ae49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572516.46/warc/9466ca7094105a6a8d900312011ab84fb2265af1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572516.46/warc/c199df685d688738647d2907550e71fa5785bf3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572516.46/warc/f7c1cea85b7c90f4cb890109eab747b06c5d951d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572517.50/warc/07bad11e9f27bfdf57b344dace088aa5bf590768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572517.50/warc/43e9fa5809dbb05158566cd274b1ac951c01db16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572517.50/warc/49c538e654767410c4da7f2f3b3df297ed9e1866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572517.50/warc/7fae1a3f85f218dc5cd1b8677f606c425f67a18e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572517.50/warc/a714776bc94cf88c50b41a6eb76d43c344706a39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572556.54/warc/308b45f483c3b50edd4dbe3594f2efddd1784724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572556.54/warc/b4ebf6f8e5e20d93caaf35637b734657d0e0fec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572556.54/warc/ca09c3660f66c547955750de915300d0061c8d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572556.54/warc/df72e2a17bc589fa750b8c44b30a9d92e5bcae92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572556.54/warc/f312e32b5c2ac875f18dfbd21b72a8d7cd2104af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572744.7/warc/21725b009c9667947eabcd7a10f9bd1af540760e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572744.7/warc/2e6486ab966da7af62a3af4a0b6e8dc96e2237d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572744.7/warc/6fb8a52e99d5ff04526591131b10abea6291fb9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572744.7/warc/b1b89b51e57c39797bc039984037fb2cc105709f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572744.7/warc/bec4cabc4337cf8ab17bd6992970452dcc5a6d89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572879.28/warc/35434d4eda2c6835badd581dcdf87a04203bb55b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572879.28/warc/57909cb3f9c952d61ff1db4a83911502d0abbf05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572879.28/warc/859d33c31ed984e4560f3e9dde5a00f884bec0f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572879.28/warc/a6e71c5d51e3156c30d10ba50b5a24530c8bb6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572896.15/warc/1640371c3aa73b172d06bb8a26fcf3aef29fda04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572896.15/warc/4b81beab8074f2679b51d24bad2550a2ea64f69d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572896.15/warc/809d432b694012a04158afd2e7706a2fbf643821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572896.15/warc/8ae49e14b0d18a422561de003ba4ffedc630b169.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572896.15/warc/c26eaabc708ca7f2fb4ac89b7a1617c7dab0a686.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572934.73/warc/1ae9b09c2086689b010abe3faf579429ecf70589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572934.73/warc/5479d2bf1cbeaffe8437a8099b59e191792099d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572934.73/warc/97469f914e00755bd1b94f9a13da0039c1fbfc9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572934.73/warc/b242e4dc96100972275a6594001c1d7d44f0b701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572934.73/warc/c85b239986c83cfa68aadfe0fadb830d7f8308c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572964.47/warc/006dd676f5823c4e769fc6055f586fcdfd482b75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572964.47/warc/0541e7ca03e80e5efef75e23493ea7fe7085eb29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572964.47/warc/301608535d3b37b6a3a8907b08c99fa6514c0152.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572964.47/warc/87e0c64334d141979944b131a483d280a49c1e98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572964.47/warc/d8e078a10895938afd449fd6317cdab04eebbf26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572980.56/warc/0477d01ca8166782c4e642b66a7f20b5c84c025a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572980.56/warc/68da8592304865db3948daafc56f63712610f100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572980.56/warc/8a485f5bcb86cc26e6b3fc533294512227270e73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572980.56/warc/c017ad25bf744ef72a3eef960e4e829aa8193356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572980.56/warc/d560cc3040312f9ce5b3f1f3b1279ca48cc449a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573011.59/warc/37497371540aa07c7c72c9c29aa0e392522f95c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573011.59/warc/cb135ae56f709764881aaa9a6259eb6f4746d1c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573011.59/warc/d8ea38d473fdd2c69e9b8f23320bd417aaa5c66c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573011.59/warc/dca93a365de8b2e1d81104beaeb607b8b813829c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573052.26/warc/604016f8095241d436bf3e4137351bde06ffdeb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573052.26/warc/94bc77bc0a78216f1f829b6c3fd9e598f6bda7ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573052.26/warc/a8abdacf43daa559f265662eaa24832848b4d465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573052.26/warc/b3bf4ec8f5923e66bfff3bbb83eb7275a1a933c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573052.26/warc/e06ebe4374026051d88c5e1e3f469768c021d54d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573053.13/warc/11209c2090bc2df57d37d3c502a663c2c28ba9dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573053.13/warc/4360bac1558c962b7e7641fb212a33a24f3dc701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573053.13/warc/543d095bab97bca14ba40204037a962604ad8ab9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573053.13/warc/74b375f83b7b53b8965515b70ad252daaeb8871e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573053.13/warc/9b9117036b435cc724cf892a97e10493c7994e9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573065.17/warc/1c4e194b89b2f5613a297c182e19d6992e7533ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573065.17/warc/2b5851573456f5bc1186edb31b2fd496c9b72987.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573065.17/warc/3c4f4734032482a81632d7875bfeb2f250cd40bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573065.17/warc/7df89c938935a66cc99790df1cd323c90909d2ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573065.17/warc/a86042532e86b128efc3b409023ff7baa4369792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573070.36/warc/b5f647161c9b775189e9486ae53611324c849269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573070.36/warc/c5b30c3c44e52e99f3dbf30632b304e08988915c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573070.36/warc/c95f4a1e9172114db05d79c4d07d2ab32c50bb83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573070.36/warc/ecbcfe92924eef121fdbf19f5204cd43918e1a8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573070.36/warc/f2d49f831e03e570fe2101127b10e23e12e8a39f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573071.65/warc/630d388cb934f6a6577367cc01b6daa4ab006a5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573071.65/warc/782b0862d26e2df872ffe88efe9ac54cae9a6366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573071.65/warc/9295646fd30d63b56a21a0c5e6511d14f122d65a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573071.65/warc/fdbabca1340ce6fc245cd0a0e199d545932ac3b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573080.8/warc/1b5091951d46f31b78e3005adb0015b9c7ea550e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573080.8/warc/275a91f91cc38caafac979995a2dc3e32ed91850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573080.8/warc/35624534eb8a60101abece434d06668b4223fb30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573080.8/warc/46da7f7ac33c2c63f3d60f1a03a7698bf56f56bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573080.8/warc/770c2878fcf38ecf47e6bfa97d275487e00dcdd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573098.0/warc/4f448c85554916aaccbc4a842bc7ac653e7e16b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573098.0/warc/8ab4fab0b324d203372783fa3a6f4a3f6768c3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573098.0/warc/8fe6c8e7916d1efa138700e6ac869f5f2d56d98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573098.0/warc/a9d92c78b14e4730171b02fb13f4ec0483a5689d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573098.0/warc/fa4d253240b61868e7716b2dd332a7bd161f6198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573105.1/warc/53014e0261b5617a3bff5e95191e67fb08ec343e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573105.1/warc/6c2cad07a10772ba79a8adbce46b6b919c2c28a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573105.1/warc/7d398fadff0762b8eb98ffbedf466986ffdd6d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573105.1/warc/af1dfb8a84511b511ba9914833a0b369d9af281f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573105.1/warc/d9f7145a99a6b9c44256a558ae549b16e730267d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573121.4/warc/42002b44627425f4a5fb06142ba3ad8683b6bd15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573121.4/warc/46a3d4d39a19d31c77a4207ced19bec35979b7b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573121.4/warc/91f6d942e5f0f5f3a1010ef7dba7cb0b6fa21bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573121.4/warc/d9552598769defe5c2e0e7c9a5b1d190e5162b0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573121.4/warc/f3bc96712cb3b1df60e3d861638879bb3480dabf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573124.40/warc/1034a67f23a8483eb92ccee3ea41bba033d1dc21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573124.40/warc/6d8cfe9d7bec407a6abf750ac9b0961b66f3aa9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573124.40/warc/9868d15a3f34fc2df5f75b6f04efb520f5e7a2c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573124.40/warc/dccd63d9e038f0bc3534b9665de97848cc042a81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573173.68/warc/0e7dae0a1858155f4272e51c528bd1c82812472e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573173.68/warc/559d418ccfa4ffdeec99268481701cc6ebad55f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573173.68/warc/e83af7f801fa8ad5cc12c41463cd95d50e5eb4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573173.68/warc/e86b282f8eefbf53e3da4d5c22540193b3ae3035.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573173.68/warc/f9dcf73c4d703a5e25551a593dfed9b3b6534de4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573176.51/warc/1d08706d296d645cbf4fb61099634a89d4efe3dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573176.51/warc/59c786f933099d111f4feecb93358060906e03c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573176.51/warc/b15f46a173cdf51820eabaf9a4fe93d23f380854.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573176.51/warc/b340a026375e98f003f38329fce03bf3ee26e610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573176.51/warc/ea92b0ec9583d6c2773c0d353bfedd47329e8214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573184.25/warc/69286ab2c1bc9a0d23c278dcecfe9ec9df9cd256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573184.25/warc/a05ca6f0b7cbf884c387a857cdf11dfdc8eb0d7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573184.25/warc/aa77bbb1a1ba2e41d40e1ca9228e4658d836a646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573184.25/warc/f1b43cb978134b0e7683734c811aaf7a67768850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573184.25/warc/faa99ba7e68ff60e751524c20fe48ba9964dabf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573258.74/warc/0211671fcadc2df12d9a2f541b4ee6c2a91b9e8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573258.74/warc/2974f77eae2cb3912a9f2d25e12f1e0a6b1331d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573258.74/warc/88bb8ec1461f885067e8ed2183a9dd3f32f55d29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573258.74/warc/93c198c1dfb956cd46c7d837953b6bb843505ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573258.74/warc/e464cd8b15c7f3ceba323d8ee3d6133ccc3baac2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573264.27/warc/1d741d373000091d82499c50f5863dd5ccaf18f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573264.27/warc/24de8aaa90ae0304d063c8f6a57c08d2d1464007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573264.27/warc/532dca4dc98b32cbf1467c395c875c392ec42877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573264.27/warc/88472dbbfd674870f6daceb9cc3f481612a16893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573284.48/warc/33277ffbd71625e0926369ddddf943f7c0832d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573284.48/warc/4e9e73983017b50de229ddaee60188232da94f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573284.48/warc/63b35eab7c46c1f62f3d67b9fb095bde1fe55cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573284.48/warc/6b5e0e0e5fc77650d2b0c983003c2157851933cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573284.48/warc/b2222b6f411723abda1595da535d1322891c9d0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573289.83/warc/28a7f8f33ab3e4c29d6c32e8eea50a077ec602fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573289.83/warc/31b613a2cf02b656ec09b474bbefaec519a15ee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573289.83/warc/6725c063f9dcf9a571480ae5565678fe0b523c19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573289.83/warc/923b332b41af16d1a513a1499f33d2f3f1f96a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573289.83/warc/e8f8cae8c491ca7f8300a8d699c44193994d91a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573309.22/warc/2e65447479830fa67226daa6d8e22e6a8336a051.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573309.22/warc/53c74f880b0f7ea7e06276133fb9be46af4b70bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573309.22/warc/a9d320e0a45b997c182ad9c8af1d845e6a114505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573309.22/warc/dbdc234cf46d819c632296edb8773686a7e18e34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573309.22/warc/e3f1ff531132e3485620e027445f6d12ca85d147.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573323.60/warc/7dcf006fb5f5b4e23ace98a886b9c393aa8feee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573323.60/warc/a1662ea6bfd14b21783b0d25051bd6c9449d0375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573323.60/warc/d110a156660bdabd1fe75f6c2f7b6a5021d18154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573323.60/warc/d1809db0664eb56202d34eb0419c804733321b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573323.60/warc/e163028c233990af918580476998d080c3030327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573331.86/warc/096afde3fc5ad399e4800f47b9df819a0369725b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573331.86/warc/375900c13aa473e579f46ee0bb43077336490e3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573331.86/warc/9cd2722f1d419875aefdc3b2ca2bcafa661c54e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573331.86/warc/c46f1454f5a7877369ec9e2ef185e4d43dd6e13f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573368.43/warc/2341966aa4999b160088087f59269d2be32e7009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573368.43/warc/583694352b31c8ccd5a16d8ae30ca56826f8371a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573368.43/warc/5e4bd3fbdf21b35f79211a6c109e109ca35267f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573368.43/warc/dc62ecb5e9ee38641fec7756d9d124cfaf1ba727.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573368.43/warc/fa81b0df518fd2f6fee10a87a2bd1abfc7b8390b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573385.29/warc/36b7189d8687c342efe1f50d99b254fded970fb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573385.29/warc/36dc9b9de16f70f8a28c78fe0907355bdc47eab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573385.29/warc/7ffbcae5fde1efe7c59b179a427b00b0507e9984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573385.29/warc/96b76700a09c059fda85356132bbbd787e6193ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573385.29/warc/cdb7b8cd28a34b9e8ceee6530de09c671fe238cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573415.58/warc/8122edb2f710f06e1238249b0a431b04f30078dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573415.58/warc/8e1beecfa40c0bc57409bb1f006487a48f118cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573415.58/warc/cf20934fe2aaf7cc0b0eb9a87463e4ceab859c6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573415.58/warc/d8280afab6bc59c63d8f995140ed9424b8aa2863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573415.58/warc/feec697dcf37eef90ed843c2d97f6241dc1206cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573439.64/warc/1e3d8ac390a015e99d1b6515b83bcc28dde0b139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573439.64/warc/4a38afe9cae8f9db10f8f9e3af20662d99c0beaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573439.64/warc/589172243df3607497fee81b36ab3fe6667902f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573439.64/warc/7bb51ad002aec452413e23774cb91190e729cae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573439.64/warc/f42b1213787464fb3dc7632f1e0cbf1bf7de1182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573444.87/warc/7406b3b812a4966186566b6f049030c4897b99a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573444.87/warc/768b731a9a661da18aa6823cd5724ccb99e3135c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573444.87/warc/dee78e8a4c949d28091cffef777de0dd60dfeb1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573444.87/warc/f8f82900e3d19b01be6a774d8c2a8fddccbe4849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573465.18/warc/1310c6953ee46bf900b1de3d46069345421903eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573465.18/warc/233ad59d75789b8380e56e93b4b5bf79f12c570b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573465.18/warc/96cee1948ac8eb0ccefe1a5137579f429b3accdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573465.18/warc/97450269b2b33dfe60e0f356f32bd5dd3b95047f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573465.18/warc/d6d8edc5a56c2a59148c4bbf7a7c9b4208b389b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573476.67/warc/2992c8e5964c7d21f8e96eceaa638e8a0f3ac368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573476.67/warc/5734a44ece111ae75d7a3b176bf7a8dcac578ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573476.67/warc/9027bd356d97b01bb34d16aa2137d1af98b436cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573476.67/warc/995c3f6b5aae252e49040eac5e301aac7cb14d6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573476.67/warc/e6370e7cbab35df740fb9d0ff267e1b271eeb107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573519.72/warc/7a570430392edd90e7fe61f103c026718bf8c5cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573519.72/warc/87c43b7d268058afb7da71913a78159d67a20ece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573519.72/warc/db019f8927a15e985f6e2ab6123e442262a52117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573519.72/warc/e940d144b4164faa5b73dcc43db165e7f1722b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573519.72/warc/fb4fc1270032b72d0fdcbd6cfb24eb50cddcef94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573533.49/warc/0d4946937ecfad788af5198d60d58d7fb2631d41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573533.49/warc/797ab92f24b79ad070a9fafb775c39a7e5a111bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573533.49/warc/b2324eafb773bb542b8e51f3327586a346f2c57b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573533.49/warc/be82b26a9368ef2d0c2911955f0c7687896a0c5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573533.49/warc/e1b05c41576c2d6df2da7956d808da940ce2be7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573561.45/warc/174126df2ee89871dd0b309eb50b781749b980ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573561.45/warc/2d31100769ca216c9f2a2e2afec0f48b7a905157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573561.45/warc/38d041945f4de5b5fff37751677022eacfc4660e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573561.45/warc/db083321ad1fe6c7701a8bae28a3c3d930953dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573570.6/warc/239865ca27fc5dbbfb485533690138d74c725f71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573570.6/warc/724c202261a037d3111de440b09d2ad14a40aa55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573570.6/warc/7ad0fe29518bd4c4e0c4ddf665520a589ed057dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573570.6/warc/9069bafdc99cb0584c22d2705d5a87d3562592e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573570.6/warc/d6106d10b37053be657838d3a6f7646c31f25550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573735.34/warc/02fb0c42efe5fa407072ca32d0e0c27da779ba44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573735.34/warc/33f1306a14bf07204b63d1f592d0fa8fbf2efe19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573735.34/warc/73045005142585883cfdac34c73707d0c01346f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573735.34/warc/e84e58a150acbfe378194f8fa7cccd23bc1c52d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573735.34/warc/f9003313ec8662504fc88fbce16631f2519fe49d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573759.32/warc/1cb8e251d6a487ec8d1246bf0dd1eba1f17fa416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573759.32/warc/242bc6bd0eb8585f69f7c7b8629651dd8edec9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573759.32/warc/81674d634f3965ad28a787d0e2a1d61b53ac69e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573759.32/warc/c59e129979ba9dc2a5013b0165153de401f652c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573759.32/warc/ee0554d6983d65e2ff6a584e301a08a0754c50f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573801.14/warc/34800a55ff1bb3b27d0df95b9f309fba2e26f49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573801.14/warc/6b3227b24d4c963b0c9f70aa8e94e1db950e3f5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573801.14/warc/73aa29b9bf7064d072fe89c87bb72b3cc633baff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573801.14/warc/8031a6d377ce83b79de8645049f2676ea409ff98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573801.14/warc/df2fb74631b3c03429cd74e34f56b4ae64c6a089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573827.2/warc/35591068bdf38ef3a6112c3a32e8cc047ff7cf9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573827.2/warc/4cb1b50243103e720ba009bf70d415216d9c6379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573827.2/warc/81cc815a75b3ca5103df0e95f5326d6ff3f14cfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573827.2/warc/86909cd3165088320c080831c103e17955893cf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573832.23/warc/3829550e764bb9bfe753b628d370ef359241933e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573832.23/warc/51079586bd776e213aae113771e40dd703a08060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573832.23/warc/9eebbaec0aa676aa16e2eb9e5a47f794112c7f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573832.23/warc/9eefc879ede368cc9dd2dd945936b1d2ed7e9d27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573832.23/warc/acf7b081ab2fcdd0e88794c68128e605511c80d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573908.70/warc/56f2d495b946dbafe6e41a10beb439af67ca7f2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573908.70/warc/8d72af45bfd688154d124447ed538668a42dc137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573908.70/warc/b9f2ee0c3e3e26ce329bccf3c5d40dd28e254b05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573908.70/warc/c2132a04a52ec5d740a1b23f924f7a10eb92ce1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573908.70/warc/dd1da9266012864b9a29575804d3b854b5646b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573988.33/warc/268ead563d25eb1210ec2239371dd0b6c9f9bb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573988.33/warc/458808362bbbcdde28e9233dd9fe11f354979d27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573988.33/warc/687fda515d5704d7f61d4b719d0c55d50d58aeaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573988.33/warc/7e4b174e17758a99d2ca75a90072332159a68741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573988.33/warc/b7addce28c41f28702542379ad827eea7408ffc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574018.53/warc/1fc5f64114f445e1f498a785b82ff2ef16dcbe0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574018.53/warc/273864d0c445846b8745a39c66316a60bb3a853e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574018.53/warc/37bd40356d435a4423f183cb57bb41ee11ecd5f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574018.53/warc/5977d42c971ec31c6956dc06d5554a88799e7cb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574018.53/warc/61852764f05439cfff707b7bd680b0fecc97afb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574039.24/warc/30b07364927e442c55d5cb6a94859cecc8ee5575.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574039.24/warc/79c9438d00fd026a89c2d4c7d55d43346db7eac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574039.24/warc/930778b887c650ee20503d26407f651fa64d27bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574039.24/warc/d621de902cf0ca0255ff9b1ac500f90f42a1b821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574050.69/warc/22347ec2b392401e72372c085ca7423b5d714656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574050.69/warc/69a39e311cfb441da3c0314c695ec6567088051c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574050.69/warc/786e79c4e6d8b5f7d25f89d8f1a15ba777655ce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574050.69/warc/9cd7be1428d58d908e47fa0d17f0b696c6cdd584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574050.69/warc/e99a9d6aca1e7dc4fa04b4e425ac6104df46f100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574058.75/warc/268533a5c3da17051fb74dec87f9fd6fffa9f42a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574058.75/warc/6e8d6ca5a7807db62db456738c188399aaf89b71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574058.75/warc/7e7352fdd6d801b7637c2d4941bf4efc35614478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574058.75/warc/824d784e4b9b3bcd294505be298a535389e8a97f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574058.75/warc/e3fd4cd195ccb3a9866b2227214b360947865d3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574077.39/warc/567f0ddbec7a0cf8b04a6b2232e88cea5f079aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574077.39/warc/63b7c7e2ff7230fcc6d63a202d5877647b7f7d53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574077.39/warc/c0a7d58fabb2c6e6c6b8cd733f1780a4853671c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574077.39/warc/c59dde39d59bb4429661c6943e8ac62163b8f7f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574077.39/warc/dcf491e43b47be5fcf1a2d4cc9b5f452b4f78120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574084.88/warc/35030bf64d40e325210b7d91b3ec8eae90d8e383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574084.88/warc/58f0464e99f2ead2a36c1e9e5713e68c931d2984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574084.88/warc/5b0d9a4f479a184290ea6524873a29330bc3dfec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574084.88/warc/65dc61a540c12bb765bd46c3da64950837f16625.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574084.88/warc/826bb097c74e222e6077c7bfe0d971faf516bc6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574159.19/warc/1674213747124e4a11b5a2271685431245a7b73b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574159.19/warc/8efcae9a8ac30d2c6140bfa424383276f355a129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574159.19/warc/9fc1070144e432071507f7ba950375ab534dba43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574159.19/warc/faa769498efd12d03590e0550bc935473a8df0a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574182.31/warc/4e9404afc871b74fd610fb6dc25c40aff510fa54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574182.31/warc/4f1b1ef706b4d0a160e502064c1d729bc1c91616.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574182.31/warc/57733ab72bc7ea5787e1e809d58c319437732100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574182.31/warc/60c966ff2c00eb53e5c6336a9cc07611a673472b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574182.31/warc/df420cc31957b1fb1e7e417ef4dc4b78227c0771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574265.76/warc/712b9a140912358b58ae7c6da47a2d5a818ec53d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574265.76/warc/731bd4b088166ea2fafc7636f33929a350411526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574265.76/warc/a6bd59b82013bd511cd096eb882a2c77c75bfc16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574265.76/warc/a990c1258782054b0ee741e7d2447dca64911766.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574265.76/warc/ac2e3178fc67705dd0e933e2f5cae5ef14542fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574286.12/warc/842d45666cb4b7a6fcc608f43cdad9b3970f1d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574286.12/warc/8d8db538f1821b08427b58d9eb42a51ef8af1e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574286.12/warc/b9666aff31d0df870661dfa1337ef30a5376e401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574286.12/warc/c055f2a17de24dcdf2530abd3259e89c44938f94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574286.12/warc/f2d31c5d1620d1f6a57eccad4b108aa1319fffff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574377.11/warc/8295b77e52b071a1db8621d13014ec90c695dcbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574377.11/warc/9190709ca11f71476dd1b1890a0adf160849f50a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574377.11/warc/bc7c6b18cc62a496a148c42761f150a793c8df88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574377.11/warc/e38b77f23401b1d78d8133e2019381bd1c0485ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574377.11/warc/f1ca2eab9218141df55896782bf7deada69b75b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574409.16/warc/20340f54d2ea76c100e423943a6ecbcad5c3501a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574409.16/warc/37badf6a35640c46e4cfb6349b553e7e0483fa52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574409.16/warc/a3aecb7d014526060c5508f2527c026350603411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574409.16/warc/ccb0d6f5a3206572263e0272d2ecd1a38167feb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574501.78/warc/37a7b92dcd63f8ce7f6b6587ee86b47931e08555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574501.78/warc/585b385c227e88ce81e97648f0e43bc09cd38136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574501.78/warc/5af7920053ca89ff885cb8d92391eb6ade5107fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574501.78/warc/be9ec9aae41d70e29b67cf80d7a73e79cab97bbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574501.78/warc/ea82577a23c428639e75060fb047bd0aa539c8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574532.44/warc/0fbf40e7a3603f7e7a571f1f2f10174507a9560c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574532.44/warc/4f8067416644d9b8ba3757af1bae66f9f1d668ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574532.44/warc/583f1c1d7842347080d6ab85ab5a0cf709e1cc53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574532.44/warc/94e3ea438a9cab96a8c70df70541d64176637b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574532.44/warc/d14f066b085fb447fd4f7ff6886b47153bd9e79b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574588.96/warc/8f1ff1c83728768289b5c2304eb2c3c2de1e0172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574588.96/warc/b71638d2977acd2de562c27a04f3747835764ca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574588.96/warc/ba17ae432ee14aedbab24fe81b9086dc8e6c8588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574588.96/warc/cdd342c5890d0962a723139c4d99e1c8cd9c8e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574588.96/warc/f9439beebba668935dba4477be97266b846ec757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574662.80/warc/35a13eb31752060bea4e7f65e84936be89b0d1fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574662.80/warc/39ca68f7967ae543f82acd1127df73ded6267ced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574662.80/warc/4b6d717e482c055138beb1bd2710da8c545a14cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574662.80/warc/85110e20e17a3e4ccc6a7d54e30dcf20bd88b134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574662.80/warc/c942c186d4f3b8d6860ae62dc974eaf3e90a2754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574665.79/warc/441d679a9abab45171a863e9bb4facda2246ecb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574665.79/warc/493ca19a6efcadaed48b7a7749a43dd0f4685745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574665.79/warc/84abc573210e127f05e3a280be2448d75d343899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574665.79/warc/9a4f1dbd77fc2a05052366c5773b416d9dd59e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574710.66/warc/05b4f8991a227ab52fbca1d851d855dd4b01ec4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574710.66/warc/541c8155c65d75e00d9f4fe67d1aa22e54ba7140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574710.66/warc/7d6b3dd3708f581c8652227bf75f7e96532b4385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574710.66/warc/8ffeaa51e8fbc9a33d74e2486bc2a61dd7a6964d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574710.66/warc/f2cc380cfd7dc1e2d0a7fc402b0bbf69a3b70c8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574765.55/warc/34dfe5375757a5d631e60f38db96ab485e8f4f2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574765.55/warc/89c90c3a2ee41c5eed6026f52cad42af2fafd7fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574765.55/warc/cdeac2d9da335fd83f6e281b14219bcbdcaebce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574765.55/warc/e23f79b48e2bba6106b8fc96c988f44930b9d8bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574765.55/warc/e4e2258d05d4439844971a46cbc92417704e5c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575076.30/warc/2a7e4ef7ca036952b8c69f9976cda57b14d6d4ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575076.30/warc/a704ef077738d721f9eba6e96fe857d0257b6507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575076.30/warc/aafb658d283a7af3f734387fdc1575ff75446767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575076.30/warc/adb63a235895c3013ae3adb638eb1ceb39fd194a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575076.30/warc/babf35e7dcd11c386856924ac3096c6fa148f09e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575168.82/warc/03f957956c57f55ca78ded3a2c3cadfdde081c5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575168.82/warc/0bd779d44af254d119165920b2b98f8afd3a0fff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575168.82/warc/1859248513b29447b9c99152f7d2af2a3161deec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575168.82/warc/ab7390c542eb7c04aa38ba7e832dc3519cfe1b0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575168.82/warc/d04d4cf968b5156e91409fe8663e241f12639ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575402.81/warc/98cc9460a20f15449ac55b40c96e7a2130685da9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575402.81/warc/b36ed71544e0e7c2a48ae4027cc6af18a0e8d56b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575402.81/warc/bbf42256d43e3d1884a14a82772f0b175757edc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575402.81/warc/e176955a42bf556239fca29e52d9632b94a9f611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575484.57/warc/4aa752f9a877cf740b7387ae5cfc71f5ad6556d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575484.57/warc/4f1780e4eb5a0251b6ef752d30beabd2e2e195cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575484.57/warc/acc7c571d2adb27b4f8a311a39c4228e36807f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575484.57/warc/cd00bdc27ed8e866a5e44395cf50626ef498cf9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575484.57/warc/e957331a5efbb7373def46d6d8af55bf3c0733a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575513.97/warc/10be8691c7dad64505b8406d6972443acd34189a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575513.97/warc/245d5d9b4537a261b1fb694ebe57c5e7cefbfe84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575513.97/warc/41eabc21123f572dca7d370463e50db97bac0a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575513.97/warc/6d77d2589dccb01c4eba8e7159b8f921e083d3af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575513.97/warc/9a14ee0e62ed0f1d08f1d62d8f5dbe4b3e1ac14c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575515.93/warc/1f032a621bf6a520e3f53fc47ff2711831d67241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575515.93/warc/c37aee78736b86cec8738acae32318053a189bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575515.93/warc/c89df2fc8a05b8bc09778608da60964573df229b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575515.93/warc/f6cdacdd70a688fd4c495daf2ad7444407788690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575515.93/warc/fcf85ae657a3ad2166a095a24d3082188d81ac29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575596.77/warc/07df79527af78ac8063694ce00674fc25061312c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575596.77/warc/3c78c97071a1aa6b9c9256ffd57b469bbec79fda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575596.77/warc/849759e44bb0711b98f610a93f6467cd494f5fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575596.77/warc/c4e4c64f8b428c006660b89eb3b1fef8ae817b40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575596.77/warc/d5e2fd73d3f0212087020ff23601e9f472aff1fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575627.91/warc/243fd0df99137c5930383d477ae355744963cc59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575627.91/warc/6c0f3728cc1e00b7cc6fbc7dba17ac0cb51b1a42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575627.91/warc/816dafb6bf9ddb976834a5fbe7810017aae5f058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575627.91/warc/fd18411e1d43f376d27ee4ad3e0f767457a08a8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575674.3/warc/0348f5d784da7ea830b5316ee508566fc3af52aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575674.3/warc/06b710e26e7323dffdc1273331f24c54f6c53f1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575674.3/warc/2588d101e56d66249ced7a0ac7b33b6320c3e954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575674.3/warc/c28a960a085bf0dafa683dfb79715a1e95d0ab26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575674.3/warc/eb5c83613ad6fdfcbf3f5cd6a2f1fcc7b00b17a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575751.84/warc/787a57b5aaa5f58bfe18e368c7474ba7e75cb468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575751.84/warc/c374970c7cb29822e950fd9d0ba7c455e00e4f1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575751.84/warc/e27214a1f20b9b6647a92c8073ddfc38b081741a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575751.84/warc/ebe841c8c7627d8cd55f3c423b51b820ae3ed827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575751.84/warc/f2b280049e0ef60a577152eda31a980d2bc27a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575844.94/warc/052e57e6617c0ae1905a2d1c87a8fe042935e25e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575844.94/warc/2c27cce03876b14b636df598d688ab6ed7b5eee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575844.94/warc/34951997dd9342e5d217b0940dd0502952425e79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575844.94/warc/6d73cd1c8f3c724b3101d1d55994c10cfbd86ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575844.94/warc/fdafb32385c4a0b8808775eef66b2b267f3ab3f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575860.37/warc/374991dcc799acd38d3815f4b9fbb2d7651de951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575860.37/warc/5f27188d184880430dc7d8ef3376289d5a211db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575860.37/warc/a6777144c51d1a5f87ba3a050faadd060e0555ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575860.37/warc/c51fee24fe4992fdd7dfaefc4f61008a33cf6da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575860.37/warc/fe056600800e2e5e7e53c550d5c955df4dd0ec3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576047.85/warc/34055d9cac50b1c2fc5dfd13a1b95d4f0bb4e355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576047.85/warc/54492213468a1ebf0cb2b97608f2d161756287b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576047.85/warc/d5956fc55b62ecfcb951ebc9554bea1cffc459c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576047.85/warc/dc38a3ec87dfafa469ad50725f3237d7f5a35183.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576122.89/warc/20c23a735b1108072f521a28a5074c110fcc1e25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576122.89/warc/6e5d66b0364c398b87591c6e022301d2d820e5a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576122.89/warc/7c0dfa4a8efde6dc264ea23f38bfa4ff439014ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576122.89/warc/c404318a7c25eccad683126a53f619c77a758074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576122.89/warc/e4ea55e61dc659f552093475dfda700cc61af699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576345.90/warc/591da7e1045d95889582f034c530fbcf3e2dde64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576345.90/warc/6376a20c5c139488742f0299a616d3487eef51bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576345.90/warc/99f46529d4af821d9d3b88750fc6b23f0e98bbeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576345.90/warc/e22a32713814d26eaac82a172eb7065b5baaf2fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576345.90/warc/eb72127090f1e35c4d8df91313882b69dde57aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576355.92/warc/15635bbd9f4d93e8290dd6ab10300250a9fc7307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576355.92/warc/23a75b9284e2ca62df598b0b773e920cc4814b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576355.92/warc/3e889fd89448c04ce4538c8b506069b7c9afc9ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576355.92/warc/c01297f8a735fc5105bb320b3733c33dea90547e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576355.92/warc/eb87108c6863c813ee5e206da6ef51ceb92a1c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576965.71/warc/268a1c7bb7057675e147c0cb74cb6141f31ec0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576965.71/warc/bd5b61ad35e94c4b21f0b0cab195e9f559ed1241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576965.71/warc/c8cee0751f3634b83f5d4273cf7ba3f825de6a68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576965.71/warc/e55349ce49f60db40208107d8c4240eecb4e5c5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576965.71/warc/e939cac9be3838e3dae2f2eea569dc298f683bce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577363.98/warc/2aa2e29dd8d787008e8a26b297adaad46ed1b67d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577363.98/warc/4a53ef635fb4e447b7a438058dfcf77dd52c5757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577363.98/warc/8fe1089608f1a521012591a87297261b97ba275f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577363.98/warc/e9b93479f83137cca61882f8a7ce3746db43d6ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577478.95/warc/0ce6f3c5b3ee02aa4eee62f096310056d91d85c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577478.95/warc/4571b7126b2f3121e1612e868639b3bb5b0747e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577478.95/warc/865ad8d0c08809efbb8e6389ad4899641ad34348.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577478.95/warc/86e25466c50c66823467af955d58020c634308db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577478.95/warc/ec0c50dfb089738654c59414e89af8ae1b2d95ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514578201.99/warc/1ba80b57133cd7cd5da73e0c70de131af77e0373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514578201.99/warc/3195f8988f847fa5356249b07726589cc70d305c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514578201.99/warc/68391b33d2a996c7c72f8ad261be837ba9f15fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514578201.99/warc/6da48e0a0e36c386fa5186facc395b91e2a2fd26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514578201.99/warc/e3bf4ea721fb78ae81cfdab4640e209d87399c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1fa554a75fac1e825c9e78e0081aa741d841e630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1fe08670b4e7a6a80c3005cde7b2c835e316bfc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/27488a5d4a1d01b1f62ab4fdfc9b490eeb31cab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/2be5c2f7a61cd717ea6cea208043ef2c7ce69d96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/2fdb17278c28cf89c775d6bf809c2eeef4134b40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/34810a7971714c93e22a7904a9924337e2850bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/35ac5c85300658d2ef649f3a83eed765f9a0c28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/3860d7d6a987309a0fec7b3c7ce726a42b537fd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/39d963aef4b692f42c48b5ba3680e5ac87ace2fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/3c4ddb162f21fb7793ae6c01b3cd41a3a7b3f4fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/3d669767c7ac4be36a025ef9183be814b4c4eae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/42f9beb1ddbaa70229629b48434347f1413ff068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/4a39464ccee01af8751cc7395d1735509cfe7027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/4d541f4704becee16ba27a1fb3011b398bc9b597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/4d63c42a9c09aaced71aa5450b489ab5fb1ba047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/4d8d28f3c2e628cffa3a6d3d1a2980e89541dd27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/5795b6360fe2503ec730b518af28c54642b2248c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/58502b57ca43f74a5412e9eae23635ae067fd6cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/5aa023947dc899c6174c8567857c32c5a7778b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/5d9285185f60e7d1beb4eae78f0c1257ffd31f2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/64643f771737c639ac10e054d04fe4d9122db136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/65a678506c2c75523cebdc32095f23ec4e216653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/68db5be0a972007d939998a9b2a22ed48974f1b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/6e274cbfab08785bb54b4ad2f0e72f93c5ffb2aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/700df1173671613dbd4521e7e90df6ecbd678ce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/70c563c8c13b396ea9a1c77adc0e863ea6f7c465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/7652eb523a2b0b8af8ebfd3b15af68a03ccffdae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/7737339d6c8ad4598e605264bd02c927efd8afee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/773ec478e40df1ad8cf2c6dea8dcd79af822295b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/79d6ba65fd95f4f24d5317198504bd861c74f14e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/7aa12d39f90cd03d4d1f51c988bceca74e3adfbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/7cea39d70949eaf712b86f1cc8aba7dc50920d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/7f2f180dc26a51fab1ae151d317d32d45ad7b2bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/89e98049cacff7d0489d86e6729f9f1981a6ea0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/8d449ff0c96c45f31f49a5e7776ac99da5026b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/9efe24cb56ae960011aac1ca595fb82286ae4b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a13f7d1b4c35cf10c5368059a1741a56b64d2dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a160ed3aca40c649e981f70b7b2c4efefbbd6dff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a17275ce3dbe01001802caf7bde07a1c19c9011b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a1d3396d1b561b089a4ac01e7d762a4e3e558f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a3888e79472b35542d969fc80845f8b6dee25178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a38d31c137d7312d249194a6fa442e2a1ae482eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a6dd1c468e5cea1d5c4946801405bafa7a2d6b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a82811bc2ee749318f0f30e3562969e502e35e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a8da2cdfae4565d408f86d1b4a5f72547d98e640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a90b316d7f8ac8dfedeea1d5933435a23c58e1ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/abaa8b31d1dc3b721830ec52bd61665c8120a5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/ac17de4886c21ef92420154828686f99168e4a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/addf880dafba7066056ae68412105e033d6227a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/ae39df68f27fb5938a955fabe4a5f8919e88c965.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/b14ea6e51e297f7ae1fc2df09925549679f52085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/b2465df6de29274003161b8cb58e814be1ea3a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/b6152622730ccf003d98e14e2fc59ea543ce36f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/b8886a2ca454d906af720b08e52fac576a2165de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/b9f3fd4ffde374316392d6eb9feb1f2d4795e70f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/babe73006a6b7b11afa45da5a1860e928ab7baa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/bfeaa2bf5c92e475f190e1c0286aefecb970dc85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/c230f40f6389402c86a5de61e2cd4828fb564acc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/c43a7a13637b765f1d9177c476458df0b39a2d87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/c660a8e3370c90a1e928aee3e4aab915da0a976c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/c7d0d52923ab3dab0d94ffc36eb07a7f88336859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/cb79278b7142b902340069fa81f102c10ccb3a11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/cbfd1365d24976e73dfd6ed2222bab50ab0bcf80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/d19868be06020cb1f9b55b15ea3596d17005f4e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/df5a1ba139edc74920e5a816f2a53c1ed6ed41bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/e50ed2491d906021b7700d1808ca5aed81456acc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/e51e03e56b743963c7add253fd45563ecb8230f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/e64ea7f978a68e8b83f832f4b9ba74cbd1cbd1da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/e7255c7c7ffca79dc7bceedb21ebdaa266bb3333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/e87b9faa8ed2cdfa432d3a65b04fcfb8974d4818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/efaf18228acbfcf0b81ce52f6d889659b233c41a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/f0a8411140f42d361332bbf66514fe7189bb9887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/f2684edafe0c7753de132b3713ebc484136039b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/f56d71d977fef81c4846cacfa8a4b4357f6ae1f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/f7d0ed880dda38b98ac32add88f716810a737436.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2019-39/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2019-39/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2019-39/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2020-29.yaml b/configs/cccc/dedupe-url/CC-MAIN-2020-29.yaml new file mode 100644 index 00000000..8f9f541a --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2020-29.yaml @@ -0,0 +1,621 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/23c7057a58b0ca6927492760e738c6ed07b2f4a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/63777e8b14ea278e58bc114dce82b2cdd6124f5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/7eaa61aebd20d0ea34d1f3e6c3922023e72471c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/96f09a3b58ca5cade977d0fe606593db3165ad19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/d93f451d5a3a93013e8d4c98eb3addde0b3e6e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/eac1d252de63dc4b7b9bc4c116784a455373350b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/0cf2fbcbe4f0f8068f943c3918c802813518a5d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/11e3ca8e4265d2f2cf2ebd78ebb8484f26faa013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/3d42c01ef103bfd1512a840d060f840d143b9649.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/95e5a76081d66ac11b828d21961dbd88a8766f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/da5fb3758936e0f92cd17273e9b51f1d0cec25f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/de286fe46abe46928154897d065f38635b35ef7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/0e60cb39049cb1909675d9ac79fa58de5affeeb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/246e2bffeb1b3fcaf64895a7549bc06c1888dc7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/355922df48b2ed799ebfdc7aae65e41066d20dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/416057d0281ec55cd7fec143671fb05aca795a81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/74cb5a8683540f7145a7a1c5c9e07aaa6626013c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/a70faae53c184008f772210b85359c74af48eee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/05ef949e367a6044b12337259e90b25e9032bfed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/3a301b6e2309c70f4c6f720041ef4cc1ec8004fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/b43292e0a9060832960a25e84833eb4f636854f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/b9e1221a55b2ffe2ef97ee7d4f1080966408099b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/c1512e5598ca1f00d5314a14c9cf35d9546cf7ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/dc4be147d405258afcdce789a045ce03c3df4562.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/32623fcee1818d421ae2ada7da58401566ca00da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/58a63dd9a4a98d080d56eb5ac0d680ef1e95f64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/898764d6c1e33b37299347ac1c5477d93189a262.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/b5467cae8112408e3ffe84c7df4f14e7555d2727.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/ca9461a2097c10ead0b7a730942f4196a33f02d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/f87ee890a15bb45af2d3bdc09fb16aa6f9afd247.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/09addfc0100e8de179fd464ff8d92dd14fe39f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/114abc9070384c73aa745283e09dd67e60729bcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/3b8fe21a5bc6a97d9d178f7c137f06090baf96f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/61ae775bee74c199582755a10045eacaea13584f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/87335305d2a6bf79c7885d3c571a8f8a75a25de3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/e7fd32034e6abcc769c30f5f5ebba7cc42d91504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/280da6bceb9af8b0139cd094c3f84381bd7c1627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/295eead51ec6cab0edfc93bb0ec2d12c22b1ff74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/3294b4bbe39c0a9a3a78fa518a1d8f0fec317b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/3b31d245fd556364803c83918223979d51dc44cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/901b70ae1cc3c95b5fb19bd23db57bc293bc29df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/b9b183385e0a9c47d12190ac582e60b18b0e8532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/137c08391af877ee6fd5a98aea619854e18b27dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/43081661e222b09b31e8640eb34ffa1f724e675a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/4ad80e7598f08c9e58795796a33920ff5cad09a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/5d5753d6ed3555992e78a568af005096d55f1caa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/77cd3958255fe5b03224f2c5422b35e5c38e65b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/9396c08540a6911fd6c89206a4c68cd9a83575a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/15a7e710411aaca6f134143541cda9cdbbda7c2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/1d1f0801af1241057f95008a434982b00c3d55a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/4622bf1fe82aa422b5c86921431de3f3f68689ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/53806f0570659747ea7369327fcd5f99a40cc1f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/97c9137007e79b3d60bd65bd7d978e1ac6454409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/f9674490290e5cf7c025f6293d5b6dacb59e3ae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/07e35609ec90ed4fc56d2e07f57682c457ab6b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/1dfdd1fc55e3ae2664530f40161ff35fedd5d404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/3860d8ef34b71eb5eedd98c9ee6747faa5f708e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/56736dccf5482f656deed2bce9dec799536f2748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/99587bb82b8696861f6e30551cd3a6b9002a6101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/ff5e7d56fc2af4fe1ec00531c5877e5f6adf1170.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/044811eba5dbcbcf53ac23fbe77b40128dcdb0ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/06dfb1522ad0a97a38843c55062c798fd9d36ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/10db5b5b398a6475cb99d7060c9f83d74574c4be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/2c59aae43dbfe134d46de60bf455cdba8b0f1fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/3f21e3757ee879a2ce3f32a478efa5459d360565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/5b0b2df48af3446c297a4ae0437669fbbfd6bc3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/030ee4a0f05ff1dbc798e31ab9efeafb03e97524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/102b2991198caa1081ce26464c46d3b7b043e325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/4dd796fb3296c4ec2028373bcc42be50f6e5a4b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/af2bbe1d78df6013ce75812a040e0dbf7134e242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/b88c5456cce1438c16cc90a7658cfc1d210155fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/e0c2e6fba422621a612f9a10e7770a4fe5d336e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/033581c9209383643b5d911c18b4987740c9d8cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/59b1104929b8d9a2f6f1c6631a81371749ae5ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/6ea41ed74e57ba5c58491052df2b7e11edd280ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/838bfb5ff7513b8a714dabfc225fab24e62d2380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/9861fc1e82bc63bd955e6529599e0a292d1c4650.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/cf2540e97956254e432f2fc646ed805b042e4133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/0c4afcc6c3a0f5bd833a92a80ed7103d18fc07f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/131baedf3b7633fc49823ef43445b57ac6e527bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/74f2a9652019d65ceb221a849fa72e724c5986af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/9823eda2c71c06c2fa4650de036ead6516cb7232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/b8497dc3a33ef7bcebea8cdb92ab87deb48e8d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/f6697094e35a53aa729d7f405bf68216865bd2ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/1265b6a8864ba8c9e98d99c1afbad6ec326a1d7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/1510c33373c40c0c433327026f647a3450c1e70d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/c88b4e6d40285151e65e48e7479b570a62684bbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/d11c916fa89c6e340df393e9f92b799cc8780eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/dc40a3f95bf51c9c300c5929be6806ed662c70f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/f7d03c9468c8bc540411dc92198df40859d5be36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/14cf9946154805cee761af00c7f445f210b16099.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/8018386c4f066117fc017982477e63f3f73f6452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/8b245150c92fd17d57d3a8a4bc6a9fb7b107c3ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/9bdefe52b9a2416724306794fb900249cef1d7f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/c65cbf3ead4c9282af8adfe3c91ba646130f3be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/d8fff6d5936ee4f4c360bf169458306c341b760a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/0351277964bde7cf22311f267d40899d8d80b29c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/0bb49f1d1a89733e8cccf2474eef51a3b62da754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/362ff2c970d7183356a84ef78cceb611b983d3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/400f556b75be4dcd715cdff8e1b09937b0fcd0b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/60c893b642fe41255578b5292abc983fb43d248a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/9d9f2fec0ffb08a2e228d57d71fc508184d1dca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/01f9abf30eac615274382f90f5a7a98fc55440f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/9526f5679b09ac0ec7aa7f145d35002ae4032420.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/9c94d099e01402bec8a0d5837ffc1960732e352a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/9f5ad613fe4efb9aa711084ff22963176f35269e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/e6df4770d78853ac913ecfed10f4529bcb4d359d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/fd6c35a0760b56afa623992459b50bbc10dd8e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/1957b15ceddbb793ce6dff32e4a88ed751cea417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/97fd44d14e5c43bc78567166bb1bd99f5c92ae86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/b36af9774b82d2d3e58c92bb504b2bd8157f7b6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/b900afe8e1dce20b180b7a2bd9b7a52b778843e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/ba801d070646a61e438a9631ae7c15fba8caab41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/eb7305681b7f74522022fa73ac5c0ebe37901393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/07d4dba685b2558e4d40730d05e4b6f912961c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/1a4cc5a6ac6e10d9962a8dd0794da91123e3acea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/8292f9f248825bab1e496835a81c23fb6427000b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/973570265df0d1dc7ca7ea4417abbc02adb012bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/aa7fe0a1f49d21f11515e06c29b0aba5e1819eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/e559eb01eb07c49aa35df46e7927622aa5778381.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/0a0c420b61b3def1bd9ec6c42fefc65a96db4807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/167b2910ffab03de413b8d8a8f50102c7745593c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/5a8e375a6c798a5243a2a5bdec5307df0cc10c5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/6a5cc8f41e0eb3001fffc64fa0eb2826ed3e435e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/8f67f3a273b8a869cdf146fefe0c6874b5d17809.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/d3350cac2d34297a86d4152bc26131fb8038a5ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/0217d2c58243f4c589bc2afcc9f5708504dbdd2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/073a0c1a8ce655658a113a08e6396455ac44a11d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/77e1d81e3c7f098871422b4ae6abff36aa6e0d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/9e41668644d2b1dce55ce53cc4bcd5112cc7983f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/a29723ff4d4e0facfa431bd3f57c251cc3344e19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/bd5484a2f7af690680a76e363fd68b61fec561d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/0dfedc9e074073f8a9b209e70cda10397245ad31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/1de8272ebea9ecbc325b4a0b25811504d1b2c992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/618ff227e9af3592f0b1399224ad55eaf75edf67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/63c0eb94a8b52a25ce26225eac4bd616ca412805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/ac1e4abd75d5bbef224e5f3ecbbcd62c407ce429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/fd9254868c5b6e7e8a3cd45f268826d97d8feee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/0722356a9dbf6ec16a14e3fc09c2fc591a6cffaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/1a0b68b248c34f6efc5b5da4c3e7b852bb04d8af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/343c7aef7e721b3bbb9de837c8253d3ffcf46087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/5ac73c7aab5bcf56f5d0d4aff403e4bf7ecfd90e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/cee06e7762d630e894e13e15309cae7152ef261b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/dce33b03dc20e8df9ab23adcd842838d2f19c5d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/0c9ef972a76b9976f7e9b92f95ab39e0913e22a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/2128c1a4648341c739751b2219819b149477f3c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/7f891e8c9551eb5cd0ecc6f2b3bf5d4bd97dfb18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/7fe171cd3291c2602064e675d2c46eefe0006c0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/d29a15b3b89defeede2d137d8a939105f4e3f15e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/dfdb466b3c0af0c4c90b53c09faf924f6e254d4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/3005ee5a07c1ea8a599bf0ccf8e0f886d6e87f3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/5785d61a0d927960aab8caf7f3e619670cfad2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/7ef1f88688edd52a4b2a5e2b45883a7dac2f9820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/ad571186892a32011c6a6669bcc8db56ec5c8264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/c7569ef934bd4bca24d6e4776faa80edd0f5dc28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/fbf78a42753169a0d5bb1e5b94e3a3245dc9825d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/0bb09a02387d314257be0690ff6452b4bdcfb275.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/5d5ccb00baa95f65ab3d84f6dceecd49bb87628c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/6e1745e12d38e2845594a715902ebce6b3529b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/992aaa4aed9190ca712fafa76183b676cfbee911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/a9c6aa46e0d23f178822a1994a771cbe66d06629.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/ce57cdb7bb2116bb9e5e2312e99bfaf5145ff6dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/1169f61b2380319e95cf6ae92949d25ca5e24863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/52e43b34fdaa3bc9c1a5c8f6120342cecaadb5dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/9af9af5bdc6946f800e03506f4fc03b6554b3d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/dc1ac79466b3113f10e66b735c60088ef9bca067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/e0775e9fecddef2bd91a5f34cfdfa05a4098357d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/fa00ed37a992f5d4b8a8b41f67167c7e785094f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/2aac187885af95b5a4436a6d945f73ea1d45f4f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/32e2755b73e9e078d3745cdbd1bdf232f90ca408.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/5855965aa148b80ec5b07465416b32426c3d71ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/8b92e882f58b59c3727942d7f60f5f666b98b042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/bd6945144de1ac39a33649a92ae4591c74b055de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/e124affbcb4362133dcc560bc390e15426240b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/45b2dbbe471dd6d49364c93d29c161b664ba78d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/a194c1abba2e97208c1152598a4e2725efc29e99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/a712256d71e2609278e1dceeb6ef27437ad3b482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/b2c71c4480daa14440fa4ef0886ec9a5b88ae00a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/bcdaae062dbf873b453d887d54412cd45ba52243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/da903d463e687373a703cc631fd6209fa3d18b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/38e2284df4dfec7edec77b1822293ed8ec8b41b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/9692a1dad95b4c3cf3d9a982a3324a3db8df6dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/c11c5fa6958cf10a8a1d6f65c704d627298e504f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/c2931a8944cb41b8745c62f1caa194f1902962c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/f3e7abf1cb1ed2dfe85bff337bb6e0da02ddbd00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/fb685eceda2e8308824b981abd0c585e30454ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/2598e11c423854dacff94b88919cb7bf7e5efc25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/720ce904b772160b92f777629646807a0e945a39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/796632b05383a6e9ec79a4e48d05a03fdd13c2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/9b4d7bbe49edf47f0509458d6f613b1771adbcf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/d1c3febdc62652a6c1d5e4cb051a9ed6ac38c41b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/df0d0b95ae9fd9ed5d732d9aff1f3752336abdf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/1020530a0556d9be3659e253479004b5d61fc180.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/24635db4292749606844eeb71760b53b5572f4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/64d4324530684196615741c64db2ef90e296b6b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/a30eed8076c1dc84de8f7e81d17c5e42cbec8363.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/b798a5dfcb95b4728385bf3d8cd2ec896bf1ad1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/cb476a247a2feb236ee06e6e174442b61458a1e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/0e346cbca1add9a1b81b11ec4c38a1744fa9696f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/145f92b017164051512a92fdb7f32d05f9a29bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/a5deb3236e41e37312faeb050f2bc7724c1a6089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/b4f9dc5bc1676356600b2a520088b388f7fc3008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/be3758025a51e3f23e670c2771cc7f0e5fdcd843.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/da4e9b4b4f83b45d851f0208711c457a952c195d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/5148831c427814cd933d32b4507b04c544e7272a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/950447c3ec75e9e6db64ec1a3dfa6acc209ddaca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/97d6c103e6ece871d256525e002c7e8a05e98ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/a4273294bce2e7fa23656f0f8434d3da723e8cfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/f8e52907a6d6554a9e86655e437d59bde6bc2b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/fe88ed9adf796236e88c1931bc23a760ef6ee3c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/2e232bf09404e8f9c4b3574d681a9ef71cdc99e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/39afb30bd0f66d38bbc21c1b0f5e77ed8144b232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/3a04c374845461da5e0e5ebc04d202c9ebc9c3e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/b04cc656847149102c08d0f5602e4ad39d7a6932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/b95373887280dbb25c34f6598e3ec6c41857aa36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/d68325bce9dbfb40411aeed3e8fc796968dbd656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/32105bc5905c520bfb5fa3a9dba857a7794e5476.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/56027f0616291a4ba470ba146d784e3920f532f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/6a5d7f1ac4f3eabfb070524a6d0d6fc4fa1a4241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/7a572b2512b4adf7e7f658f661eb85fc8fb52d02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/910b870e074bfaa650719b3f61d8acd5f8e189fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/a0086dd41914b0c8068c098a61db0c8caea0fd97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/06feaca411545b64ff52bc0bd9f94e80dcefc645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/76cb33cca046c06d963056027f02262ec800bc76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/8484aa4a225d741b82bfd9f1d796102098f4ed10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/86e4beedfd28e11d1584f799ea04d8f7c1025b9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/b9046973567d00b2dcc320c80f81dc2e499bbe3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/bf72d4deb696db747d027e3b277bbbd09a064439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/4dc366099d22f6499c8ea67ca9d28b299996bc38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/6d883890d92c27c9338541f2db1ed8282a8a3477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/7efeca2b5906da1b391ae27857b8331f6538600e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/931e71f8715498a266da3309a2d23768b1675748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/961b893d5f42e6ca1c15adea2e459677006461f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/9979013f919396b86ab8b8615df6215828ba1ccf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/02677b94b8c4452732d6bf86096d1942778bd088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/04e2b29829f299e50cff5b167b80855616ee1bff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/7ae729fc4148d73a012eebcc170b5246f35f9f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/9e4db355375a22c677d7ee929547c78d509e0130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/a227fb5f0778d72690ca397dc6327fb5ab08281c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/ebe7feab20a048db892c3486a9565c007ab2066f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/269936a865a88740b7c5c3ac8e09a2f4f3139ad3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/2f0f4e3b6181027c58b46de02ac0cca8c4a44a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/66aa4c958484ffa7b53b792aa8fcbc2abdecb0c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/6d216682aa703f9c37c5992e1fe0b6363ec5a98b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/90a9a35eb22954f3f86d6df5d4a4370b369c560c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/ad1eb86047b41d9c5511f0df85c579b06d890b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/2a9ab6f75330ddae4973d692b658cea235a93b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/3c1298bc1713b8b750cd4ff5b9e0b2284fd482d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/5d0f1c64371ab9a4c79fa4243adc61cddb669eae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/edf2aef761628c48e7dd823f2e8e9d08aa5840ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/eec779a2b2acd1f6ea6e94672ccfcddeb8100cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/f7a6aa1f90db4cc5092e915ce2b70c7b6ede1291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/531d9d7e9d2613a4943c08baa860a76108ce6c29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/59338d907acbe6687daf69bc380fb354ffb9209c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/5a5b6bd8226fc1d6629ffd289ff3c836948a07c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/b56232b82fb0accec1f988a366af4d0d18fecd26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/c0122c7e27dc6a1c249d63e0f0d243fe3118cce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/dca18a1f92bfdbea9830dd17d3a5f0dabc5f7511.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/004c73d54cc058c87925bc8b48eb2b8967f9af51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/02f53923268448481711261574e9634b6e40cc6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/0d9b7624fb15e9bf8fa5bbece79bf52cae4e8054.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/28506a211f6b3c90a07aef569b60c10653eaf94e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/71d8b47ed310dcd4e3ae7552b3a3535a9aedaebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/7f70d568a4bc3a03a8093ea62a07b8a8a1d78e9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/0cdb64d7547d3d594d58798b74f00ebade000534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/44276dfe4404c57efdee38c999c7e9f2bc0e298d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/532c0deaf6ee8f5953b6894c8220bca08c421944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/9892fd896f51f0b7f8407ce7e0ebfeea3002981d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/a9385083ffdb30ad69c635e6ecb7a315ade50910.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/ffd0f9acbd6327faff9492af3b94e69661513108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/210269ca04fcb771b728ae54fc2ffc42877902ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/2dd8aff9d9ad06d7fbefe8428a96e949d01e991f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/4fb42aa7cc9e9d584568f1270b2b17a52ade602b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/567ae1f55c7e47e15cd56605f469f24e59fb724b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/a3a47c330b0cb0b8e04391b7f868f21cf43286e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/bbd675545ea3efbe2222c705e12f8a1ed4cd4db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/19fc6af9fa57804284efd26cd0f5f10c1f6c65e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/1a7c1160ed3f89a795e04af0e3bc71a1e0bfd7ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/22b86b861b9d2e77072b82124ae50586a026590a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/2d228d0fec573615cce7ad16701ea846f1911369.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/c2aec460d2e4be61d1cc53209f76576824bd547e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/cdff2479cb733e2809ae28c668af0e1b76694a2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/39d39d3b4d8e40c54a01c2870760b3c8b3a0ae8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/6472134e0aa9f7baab3a20a296951704eaf0d330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/75bea127c4262e6e564a67082490672077b1fee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/e986f3a40b091eae487e5952f3816b4ae9fb9433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/f98ff05d4f023caa4a5cf2eeac1c1afca9f91be1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/f9e8aaa7bf5d1c1059302af81389d163251d1b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/40e93f4d8f5aa8a34c36c71d919315c694911b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/4d06d51b6aabe5403770a049a4a0fb24d3d58ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/599e2a68094b5e5cfa446c655169246982529e4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/7198fc391463abde037e5b540005fea7417772fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/c1e151e0f48acac3efeafc348e0fc4fd28365f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/c63a0222c0982ababe478ef3d6870b9a00f38d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/15e8f846a4d07365d56760e882a74dd29516a4b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/6a4d0dcfa20fa9f1c15ef708cec4d1b2d889061a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/807b123bc2c75f2f78e60797493e686cddc5f7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/a0cba5e68165a0ae7bb617fcdca57405fb0c53af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/cbe6349dc2809d8c58f2d90cf9d86f3bc0334d7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/eb2abdd5f6e31b6a473286d9b4b5e3c0dbbb90f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/5a547a62abb51d471252830c56357cc78da88717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/799a5ac4f77b4798ae29cee994546584dbecc95e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/9c5bcc4f6805b496d1b22afdb82912ec975123a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/a113af7606b1e619b893f515c24c62a4f7b9b9bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/ba4d1ad366e7a5cf1264f8a9279979e9500676a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/fe537f21794641f557b664a6e81f15c7fb774929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/125dd0be1c8184341c948cf8713a34cb70eb4f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/43b61a075425ea918ba4a6aadb136272a5f778a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/6d2bac322aa972b63ccb235987493999359db62d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/9537c851a40a02718a0dcbbfb52323a4387b7d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/ce5d1f8ea2d998a50492e78114f75640c79f9708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/e83f41083e9667210c462f3b3b1c360a460da8f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/111091025608cc6bea0eb56c4bc0806ca158d5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/a287585f4823899e7fd587c3596aceb38b6fc27a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/c24f334f6715615c2efebf0b2f246f177c4076b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/ce25d56351d095cd8da18ed96160aa97874bcc09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/e93a0a8ae7b8aa4e02bd7ef941b5059912744227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/ea3b9cbd693b4101d9849e7bb6e857a0f9869fcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/0bc69fa3cd6692bbee1382d760f8ddbdf69ac289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/1511114762323fed8fe915175fb7ca2627f923c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/67da3a863c7962d8788843bbf2fedaa109612d86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/6b269f460625c9d3a88614f7618fba9d04a779b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/d12bb304840cac44c927d31a7b063c35b25ad364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/e781ccad37024417c3cf2a79854aae45c621cbdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/10df3c47d7914b7a2a0950d51e05458549779241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/120d6b91c372f73abc09416c5cee9c496d86f1e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/4398ef8cca6a5e9df6f0c0cd43327947621c569f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/ad7db038e7fbcd62125bb2093f0e327d450db726.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/d2220eb7caa4c4bcf74a50d50de8d2352cc215e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/ffa4d85766007d8d54d8cb5038150bbc90f7647e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/25143ea6599c17ff94d3dcc96d06493ef90b485c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/30d987c120cfd2170911e02c42e8297c647ed7f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/33a477c139a7d888426fca614ea822b0fda822a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/5175d2496290aedf3c4805b705217576d683179b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/6114ea1146c25f9c36f80ea91f5dedb822cf5de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/c2276c405997a7a66b35d4e8ada1702fd5f39528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/081e95ec1f7b9f3c1df6fc45aec17d77d82f19ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/09c74ba1284a66e86c66a5553cb0b0924c0e9e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/2e1115609fa2468306b4868b40c49fd5a196b4b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/5c5cf4d0d07fddef82b21ea15814d562c502af9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/871177fe116f9fa6f6a2de20aa897692d2199477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/87e42a7a8d30b4d6c2680f2601dfbf775778e0e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/1e0f20984c8a7c938dbf2cdbab04dc8eba6113b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/2389e10506f23fa12814e7d1babef5a0508b211d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/2491120fcd610aca9e45c04a55530140dc070e31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/33e2685f9a711a945eed6d5522a8db9d2cb39504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/6574a7a3f767aa1f7a19cb02ad2cbd5a2991e579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/b51f8122b5fd23f7e82f2e83268268b8a61baeae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/413221c4b0c81386fcc8e3f56e0aaae196a4e337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/7d0a212a4e8b43d5b618740cb2f647db6e5609a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/bc8bbcbc26a0429cfcf83bed1dd819c6deb3db57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/c074486ffca1b2aaab05a296ac838c0148426012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/d1cc682352d5a6d9363d4b9c6cf56e2683a4e733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/faee040b57232a9dc2cfde568a8a80b087eea7d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/1f5f6e8e0b6a3b0fcd6ce9980138ee6807f1f665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/289e75ac9594ec3d20d49d98d61a384d5d061f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/758ec6efbef7c3141b4881427003365e30e5d4fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/7bf06fcfeecfafaec14cae18a8b4c16c97bc295f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/96440de4ee9e167f27ca14f40c351473c67f2646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/baee03fbb8a7646945913bcf8be3f6b781a34b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/6c19615f8dbc0b334c709f1e310b00f4cabc9fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/6c57580ec0585256153f98133a53290607aeae9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/8dba864f881c59d628e9edabd0e6837f2c68faae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/b730b442e031504aff91fc3cfa4872b3c56675c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/b92ecf389c0eed301d5c68725a10972b19d58bf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/d33e60df8ce00d40cefcbd559f08cc9775e9e96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/3b214da96af19ba8120b6c2f9fcd23d90e79591b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/7a3d8f0dfe2d5bdd37c48c8406d76ec57c6dcd80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/a17209a0f2d36d98f76213a00d74f2f0b734e164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/aecf2d1adf8e7ee9769e74bf7a15479bef34e309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/afdc4f8536d0d9b336a1a71c17c2480bae90c015.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/d5fac455b25d1cf39ec9d20d5c5c9473d8900ffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/0b99441514b22b1005b63ee04c4d269792e6e5ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/1e2da7d0adef038fef5bd9727fc9b27b0da8c973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/3ea84adc29f35d769583a188ef0e3c27b3bb654d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/5037fc6dea1dbd5670dc7a8c1062796bb167dbca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/8e3ce9cb0dc262e98c818972a5c388148cbad9f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/c1d387c41467399645657287f974ab3a7f3862ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/0548f79359f52c7f00001d04447be48540c98159.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/11d35cb0bfb657b43439b038a22370eedac166f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/2efd35842469b0695239e1828a79eee89e90020c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/84cd4d4766ce253db8ee4410725c456658934895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/96251c56f4988d72598a770410f4008f0c48d29f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/a6aa49542c6ee5296c3d95baed399671ae5f5346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/0674964aae45cbb99064abf478375d2f30051ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/384d8deec0fb68fbdec6a138c04ca9dfd29e8776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/a1d7d557cf49a2b47f347aaba6ab5c97f0b6b697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/a2ac24985286031b688983c4e3d9d79553a1e928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/b9385726f24de872826dc65fd4652056911a3961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/c70d1ed44c6a76d8c47b9160a5fedd82185ebfb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/30beda96af3e7a44794cd875aeb3f0b32e068d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/4940710627852c1fb74e788a60aed54e80afd11f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/5bcac1ec2d2961f4b6ced78476c970d4da3b6474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/6e1a9b8355b1db611c641b4a1063983c71abfc3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/7f64db9abfe2464eb0f5568104736d04449393f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/c4f0408fa036f519facdcd168976237d3fa5dbc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/4e51d1c78efa833a703f9845b0f4385a4dc1389a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/65bc1594c2c8a7013faf1580638c5990fb087601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/707309470b9a7cb6cd19c755e9a81151bc36ea6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/8285309523ce8761bac995eb7f3e66dbb1b9ffd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/9939062e93156cc75def068e6e0211fedc2427fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/c70069939d48d1c10fd10ee8ae62d016bcf996d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/0be96e94104ebc814aa6e8c81fb780715acda0c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/567cb2e3f2d0a06b559fc607e99ec6681dfd0936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/5e6cda1d79329441c35c989c328eefbbf16a379b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/5f90fcee877b926270b472260670955c86a940d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/751fde6400ffcde1e046185db0f5a37763401f38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/919d90ceb2c4f6da6f03ddd26cc1c455ae548f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/16a2a7b036722503c41d1db960f13bde24aacea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/787e81661f2f0a699a1709ab0fa3d35e3dc54466.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/7ff4c32fc41da3a44d0b37136003a3904e90055a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/a024bab4ccb77dac3d4f9e3425104435fc861e40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/c720487da1d7c8a51180c2ae2a363cac196aa764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/f2437965a1fe388450640842117798425189fec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/0d3b051dd9847c3964fb38f38ec2b645e4192e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/61a9f4f5525ee04e43bfac0d52021d7b698c6f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/a7f2c0895abb7282f56f0d9880b57b3315c86bc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/c37fe2a9893b5b2cd76810d7ae62ba47ed5c6fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/c60a16fe79767420fad42e4213a2c69de83a7090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/f9cbd9fefc01d223197da225623faa74588ed4cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/164244abac0970f1c45054921596ec61a8f82fac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/2906fb097d1795693b9c1e2f4e8fab2352370b0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/3287517f998829f83e1c664d106234d73c9d538f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/33c2ef3b382590efc881f696237cd7644661ae89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/468223988ad6acbc2154e9d6e344b661fcb3119f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/ae443df097114b757a908d18b359942b94a15d30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/1c255e9b48919deb5d1d2ebd91465519e146a66e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/4a722a58fea62ec1cce41c385366bf957f0ce463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/609c86443d4bc972a25bf183fe18e3fd431e251a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/78d2b80e2f31a4f18497bb39261556c1057e4e4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/96eb5a619cfd767796666a3b5d01a3c8a9817d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/c3a795b4f811b09c1b6c919163ff11d02a08b68f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/61de7d3d4d61486741d7473795a93a61d9b2b99b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/6d7d41bf761efc0177b1c248c1a9e6e966ff4b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/a654a856ef7db030df80ec7582d8cafe03b85fa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/edf4fa58ea1a6b56257d4a7f1221f8a48dd59e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/f5f45336bb7efb78d34a1adcace7ff0b8443ea8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/fca578f04dd1d71ec4f1a768897f465b73776055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/140a3cbe0641b561626af0d9a0d6da6d2c45ef1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/19a58326158f52eb7fe117249a89d9b53630727b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/2a507fa52d7622345b560d900f1b1141b481c896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/5ebc0d0123d0de76e84e6e3df143e0c55d53c596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/a3d43dba6515b02ed1fc84ff8a38ce6f1a569b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/f998e3196b4c7db0335f2df48713cbf2a1e9654f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/01354edbcfd42124e7ae01ecb07550f989096299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/7d6f32597a215074d51e6fd50eccaad464906f11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/8fc1a083f7d3c8bbf382a4b1220b0b7b44c250e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/9d897898becc1c7994ce103b5b8ae94fb9631199.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/adba12d919ecdc8fe4a70de7bb8a4c61027b2748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/ee07fb66a5783786b273eec498d5200fae5f4ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/54f6c934bc6ef05703603412b7f021b2f94b6bfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/69207cad08eb0e57f7187ed4429e8507719e55ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/db7a6df2c8a2f08385f48f8f8af321ba1ee27934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/e636b1c83ec22a38f3cb0e99122f292aa914415a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/e95a6c88a981cc4f8a69fe33d90a10c412c6234d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/ed7e2a4782ab0ae3e4d05233cd52895f2559a49c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/3765706c9c4d49799b98c9b0cb9e7e3dfd79856b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/4c2586ad9dbb690ff7488d1b50709f6defec0408.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/506a50d46c1c5004d84347067ec5270814a8c581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/5d52433c6e8bfda933d3a8dab3b350aecbbe5a2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/d171055a08810dfaab1756223bfe41b871d477f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/d731ba397b7f9ff1f3d2f521b90f439537935db0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/01170d17bc6b1eaea1c4bc5a3c452dabbe5366b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/74a402237a7b65a97e9957476284aece83f6aa0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/abff5ec8620e8b62b0f057cd5a6e1d42f7d647a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/cd9d82e8a33ab713ab4187d1b8eab78227c8e762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/e8b83cc2a9072be6a9e6eabf47ad3ddc7acfcf01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/f1fa7bbab63c92815c1377cbe942022e83b4688b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/4f93523c5358165b69b21fd3362666bba6b1ff18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/5ef14439fce2ec87b345a0aaaff300933f675687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/65c801e97db741c47e1f217b6b73d36d9545d2dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/83eddd83a235996ce7b1c6686ee058eacd24a531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/a0812a01467c1d694cd01f3f70f0b848ae852cbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/ec6e250ad290d853a8262e31af79cda814e2bc47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/9da1a02d7784d63ccd1d7e0e981d7aa22e7f710a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/a30b2c0c61edfd49a28f38ee0458838ffcc181dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/ae1cc401f869c2bc2c5afceefabc46abd9f5513e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/d6779b1eec8206daa7bf6f7d45647a578e49b21b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/d8f7fdd9258a5e6e72a32edadf05871a2506f806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/ff7ef48d19f96a9f1d61b04485bd33e11e05c70f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/3b8ccc0aeaa7a503e3512163a14bf466be0bc623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/3fcc8667f320f1ec999344eb8e144700594a7d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/5573fb1971eb66264139b625f6ba45c1f74c0330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/5df64cb024626e0b4ce51b069842417af44fcee0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/83033f47465bae721ad55637acdee7338f0bad18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/dfa093050de5383d174715032c051b8c31ca720b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/1015ff4cbfdbdcb1c2ed65ddd060e61606e6d2ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/2c269160b22d35746bf98bdc3a171289c49509a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/324348db1fc5d2781829340600868383eeb07d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/6402f01f691720e24cb15bf3b5691d5c3ffa9b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/cb799c9db8b849346efa1f172351efa04e08bc17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/d7e6686861cbc14b40705a8746f5b12e50d6c8dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/16c5be3544bfd32dc40182d86f30cccc897a598c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/22a371023e061cddb85846f16807dbd1c66076b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/8b3903b4f8a2bd3c7f4e3697445120eb40ef5269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/a80ecc02a691dd8f5fca0e0f80503590098c144e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/f247b48b53de7f46187e7b2fff903329b399cb06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/f34f786a82d393b64aabc73c3547ea23611171b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/08e259087d0ee34426bddfbe611b35b9a423bdae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/095beef5c3c193b2cd64ce87a50a990bcd437db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/0e06322dc1f1059efeb6fc275b0e6bae412122e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/54393dabc142dec3ee607d39fd833ef2cd681f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/579ab7e65fbdbe303af2cb789f9c427fa5794fb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/b76c734fa415687936b57826e804a484c71074fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/29f7877f1d4d9ce90d36acc6395baa35d7d7fb1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/379f9886471ebaa50271511bf0a5520db1ad850b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/63252740f193b65b90f3b274beb3b71f71e0d3da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/6a23dea5f0b899060c002f3ea57bdee8cf6e2865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/c1a567f65dfae106c0e71784a3d479d820cad6cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/f36b8e70ce954afc38476b25571cb6fb9681a200.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/019276f10839aaf558503c068a7fb510355a4101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/16f3723b04d2ab425eb8c2fd2153e2cf5a3f149e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/1db13daec4846da972ee3fe3ee6f8100f6dd25eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/565f53e9f93b97f95c5a888a505d22e85100de03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/d99ffd8aa3f82367fa75fd1ed03e4f42a6abf762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/ef49e9184f0290b68bfba73d8667179e66a43646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/4ce16dff37e280a7aec0e8f005efc89aa36356c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/a43b45132681b478d0cf5c536cb7f1701f49753a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/ad0ae798ca1adf31e9a706fe4b6b79224f2c41c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/c6a699488be47a1844fe37f550657cb1b680cb8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/ca8b2b8f163275a41276bc97681e04ab4351c438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/feb32d81dafec2c9a969a9cee2408a76aeb5916e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/0f2d1d6d3497958a99b6a4b313b170a2ed8dd624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/1222c2d91b85e29f3624aa224659bcddccda9aff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/1e31aeae00967dac93d5a98bfc2048ac9a24ffb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/acb9b7b9318c321912eebcfdfe1d59ee27deacec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/f0c2e5c218d017ed292b2b128056ffdb524dc67b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/f92e92638743c4460d03292494121b82dad2c513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/204b3fdf3cb1770db7b431d388ef1e78f01853e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/387a476efa63a8c91e13e0ae42be904e3129f4d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/38ac1bbe36ca2e0d592da84df0cc570b5416b134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/9fe0ca917faf5f508a3bd0a6a009066e1fa7dbec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/b80183e5dd7d0254f3d25b7c3447840591cf8682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/c8de0621e78cef47cf6a52261fd14b8eb622fe07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/26a205d4a45ba5a34fb7e8decc4130d0240aaa09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/341d731f09878aa63fcca6a79ee7e1546b4aa9fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/81156948c9c94706a33c7a4d73bb1d7c8d047e00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/98d30605b7510b3e436557b66456519497878cf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/d728ead8c56e326887c4c5750534a5fb15f3b17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/e8508713fda95239b7ca06193cdd08b2ca772336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/3192382dff4588072ae8edb8edfd33d692492b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/327404a475b3795376ba73fdcfd0f54744bbdeff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/363c3f436b1edc7e43743c61344e7eab20a34b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/376538e9b65c3da1504838d6b8d7482af025f52c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/8e757fc8fbed294fbb5f1b386a5061cbf0c37479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/fb47c744b48b97b3e6aaf020d301ca3078aefc65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/03653a3d1b5f7f56c7498bc2f5c218621cbc3416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/3539c9033b38693e9bbc606471c2ca80cfecaa78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/70365d64c12a86876e584a47b1acdb91fe88800d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/9ade0665e79d72e1666fb27cdab8aff27eaa5343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/ae9c40f9e93a833271b0053459791a9090d210bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/f8e03b0effe51b7055c86144c527cc78bb0f6d98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/02efbb551bc06ddd6fe58650872415def6ec9327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/196e609a54f38ed5c1d3dcbb771ba83795b9ae58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/490af4102be833941d76de380967c7fd0f58bcc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/89df5d3105287a74dac415898f4abf1ea7f3890e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/c9b3d80e87301bf80f9cd09bd5723d51ce0e1b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/cd60ebbb0e9f62f70ec500a259f1b82da32e5e58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/0a65309f83b362f893325030f5c15ce747c54448.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/219593e9c680bc6b7c88b754042a794354101fa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/4aac989c7d3b7b3ed6508fe4ff5503626c972541.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/8659245c3a35a4982bd18f5063f98809c339548f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/b6bb04817256ed99e206ea787e34847ae6f63ff2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/d630b3f5d7d228c4a2d64cd9ca99cf710b142fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/06c06085c685321e867f13a52de503f1a55328b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/2c26d12581e858b0de4f9f5687a4acab772e2f6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/3717c2ed3914061e8cd3eca26e236ddf5d93427c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/ab1c26c08f3ce4bef9bed093ae954b5846178538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/d9e2a47f5f98f2372ec4288c661ba8fd3c369d17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/e1a7c3509c3d97da0a711647f7b3dafbaa81e11a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/0693325e8f6b8a3b6fdbc7658477bdca3ffb5893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/3896e1fd24344174e6f74504cfcfeab19bdeba1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/3e259abc2a1e33e7e09a1a23ec83c89f5ff9e56a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/80eb1eee8b2898f675f1d9a1ad619510c5c53396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/8bc6db6a6e5ea3e1bbfd525794d804a62ab5c281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/cc0ec35718015540a022e3d3c7788b41f0e95de4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/01834830d1da75657d40ed626ccc7bd2cee6c99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/0f7de9014440299a427eaca7a50599da03a4a9c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/2d4c9f445d38c86b2decd07b5c6f3e236be95e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/917822b2a36aed4c50e2cd74bd2b296824bb8084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/c27ad40da4ed0de5b73c3b11210745bc5ea84d23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/fc69eeea16805abdac8d214c65aceafa26fa937a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/20e569a269bf6065c5c735771c5314cb11c2018d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/a1ca3f1f49b9c75a1175240f8cfbe88913c8acaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/ca4a16cf64d68596a1cbc6e2f6906a77cd817a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/ccfb83ef0ce986450ecb46eeb1909364fed9f5ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/eb5e67e623fa02b1b2c185a34e928e1af521d1cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/f007c22d829493786b17cc09c02649908fe269f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/12fb8aec1a24afc73d9b7eb1a4f5c40c81fdf552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/9126c8a8c4d4a037ce1e8a831f4afb913a18bb66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/96a541e97969cc8bb11fa88e6136f97dd546711c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/aa5b36b033e5c001835ccc261cee8035700fb932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/b22213bcbc323200e5d8ccf924badcf6fa7de3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/b28ded9db2684023112fddca85555b6ead7f650f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/0c69370687f7184860ae00f8a32c75b1680c3607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/6b537cf4efcc2a94495aab6d626ada90dee39a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/946d490253f11f1ed18a9539ed5089776d3d1479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/a054b8b38b48451ee12e38b8792d501e69468ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/aa5a96d80f11f8bbb48d340c193cb7600b2b1f0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/ec440e90a29d65ea0248e8aaf7cf5443b42b41ce.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2020-29/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2020-29/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2020-29/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2020-34.yaml b/configs/cccc/dedupe-url/CC-MAIN-2020-34.yaml new file mode 100644 index 00000000..d5041c7b --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2020-34.yaml @@ -0,0 +1,621 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/162e6d572290e304272284a46006a052329fce1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/5219c307e7ef056acfe8d6799271c1ffa6f30650.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/7be04bf74eac48bbf36cee8fe95ac26df18cdf43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/84076dbbac56ddf4e5cd6d5795e5c67305d3326a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/9bc9bcda4f3a39c885da034560f4f14a2d7b854e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/9e209312df6706b188fa19d377976654d24eb6da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/1821d9edc6de43d97a97d9f9a68bb38b6cdad211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/2a9e379ca0e7d0cb28b937fc485f231af351c3e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/a09bccb09cfbaabe70992dcc64e6e565fb2e13d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/aae4690b96749c4c3137444218b8cfb6cc2a644e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/b31b969bb3ddb77e68f1be767898d4c5559c5855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/b3e393eab7f6b5a4dc8ed6b3574d2b9c7c50a3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/16ef631b5f5ca280e0aa3009326b0eb7040c17b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/2408b85e74ec5761637dd86eb91f231b1d38a6c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/8a25a4565f751b2420010f156140c15111b89d8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/989d3a46dc7367703a20e66883e573fd7056346c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/e1ee31564c0c8232b4c924ee5c6fa42753486143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/f50823ad9c0c9b2f916bb3f54ed33aed69f4f521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/00293fa08d5aa392bc5969325cab305da523e213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/58309e13fc84d3d9dd979fa9048d42a1fda4de1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/83b59028dc78008ee72a3d4fb0b51f2eb03ac655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/c309f844bf97b73a25bf508c198f8f9a49b007a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/cf70f792549239a44cf1709322b1c005556e3345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/e11817ce4a80d578b9c05cfac3a862c7143fc789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/1ff4d2b209cd5c97d6579413b8c15f6518f6a89f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/531ea0994a7835bbd7a611a29eabc5e651b1a7d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/947d0dd55f03d80fbe224191f5f8bed016e6bb0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/a642f2a69ae5e181688bdffc8c2c6cda5d62d1a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/af0459c027bae11c488449075d87f07805c34afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/f7e9eae79c2e45d262279a9f33ef74d546891828.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/3081910f622048134d3362ed9c1d3ea6e4ee247a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/5202d370b2b8b8e87db47372fddb0f9a48775d81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/6469600523684f5c22cef0930d50702955703d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/697a51fc6aaf824bf65adffec352d925254b8924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/c5ccd13483a47f67e67b9ee2aaff2a755b383976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/f03a7dfd48850203f337456aafb2b36515ddde65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/50255944d5aac9a9debd908d24db7b21fa0b5eb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/682bd7613d8ff1369ca8bec996d21923e84aa97d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/77b73bdb1ee64060c22997225d67208adf6fc3a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/882207b501a1cb9890610533772a442805d41586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/bcf66b934b299d3e69252ed5f76bc3bd837f6f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/f71198aaef5055c5e7ec5a348c70bd33dbcd9de5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/4ca42db28151e0256c1518d7e4bb145c8d89c3a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/9ae1406ae0ca8a1a86e0623cdee1b36e9f57527d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/cab8c112f76ce2e413fdadcbf7659be456878ccf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/dd302e11a12e04ee8bdd9328314596a425370cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/f41e034d780f426ba6b7712661b36e08e4778fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/f6fd78610adb36239603241193c2c989d1199d3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/01f20081cff1623ed8666867e1b180ccda116be1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/3f1cad22cec38ff95b83875a77bec88d30b06ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/46dff7ff33530b3efa30cfe3ba790900b959a72f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/5b60c8bdb9052ff7a4495e015973a167f3f5b7a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/5e92e258a30fecf950ad7612d572aa4fb111760a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/c55d6ca97dea1603010c95eb274622883e841d9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/55035efde2ba7236a25ff4df8af8a9ccd1b38547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/5987332d36c27808d860c4969843f7d4eca812df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/5aed2bca0b2542d6b3493e2cfbb7fc4937849c47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/6045c15caaad5cc2bf8be7a2325cf2fbefe6f376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/6e6c1c3759fff66498bdbe0683a0a7cd0e297e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/effde5f5e9e84a34b48e28b5175fd7b34020cf2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/5c47628953b351ed03f846952b6f954c7aec37a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/b5f660686b95fb4ff70cbc945fa6b51b193788a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/cb2ceedf76499577985244eaee5842776604d473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/dd5cbe7d66c073d40f12d6f5ba8b413563aa01bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/e61357ae52d8bfce3bc957d4d81c50e707d4b288.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/fd5e219ad9489d806259c0d780e3cfcd8ba9ac2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/69290b6b00ec109ccf5a31bc16b047c9699d427e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/9cc2e2be80ef2ec26caafbddbba6b0848e949e9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/a8cbf1b1b68b55918e83d96caedec5152de55cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/bef35b0d99da6089435d48ce7e81ce18fee88fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/d630ad1b26768c8753dc1c5c2f0bf1b15e17a866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/d75b960b4e5f7dab0298285a19f60fa899332f83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/3c6c2680ae621acb0c1b48d38b6ba7adeb27b967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/4aba3bf2f670d6b170277361f96658eac9966142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/510b2860c6332ae5e29e084e5f9180ecc971a395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/609a79167ed786ffdb9d572838bf7f57613d5df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/6a18184c6d585d6bebb82daf0c98a92b3b2f3392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/b454979a1e61397f4461a9ec42a510bbaa78573d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/24250468212b50a48d04b0a561a4cf6b9bd5ec2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/4bd950eed164f24124190ec4b30cb0ec4844cf9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/5f54000ce9de077188e6f1a9a1df04f1295295c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/6979e2151f3b612568dee812a71d2599aadd518f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/811ec0c34faf9f10500a9edf9e3a08c51def9c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/e10733428d091f2a62864cf702beada6cad626c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/030cfc24fcd9be2835c26bb331c8f5e56a13973e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/33413d673a4b6edd81cb38f6fe7341ede4fdb483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/38761b04fd9ecb6f82eeef5d2801d75f172fffd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/675949e1fc9fe505eb388e4314cf133ea2aa5d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/eba547058a6f773754986ce0146fb9f1ee3f0906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/f459f8977c6e4d2829d08262cbbdb3a9fc2200d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/3608547340f10e415fbb1b12d0251ebed3c54b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/3a261cc7308109aac0c7513ea733ac70a9b16692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/5d01e825c967c62276db046d2b9e6c6dda3f3f1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/b9b8000261b554283f212641987fc88b7e8d936c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/d12d42a2afad54981b74cb1e531b02e12f0db161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/d521c453958cdb81d6f76e6f7893a29740e33e03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/267f1e3ff94f2de4d9756259b64fa5ad758d54cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/7c6e8315dc4cab3e673225598ae27e59435c4037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/93cc0b9d4e5ef25b69426261b71695655bc867ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/ca8e220f2a818fa31c8e61848fc55d9c83e9f99e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/f316a76eaf1305d458cc962dc6ecc4717ba8e5a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/f3c0cf50db0748fad94384b8cbb4333108bf7660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/3ce836d91f137c0a216c7993df8b6d389b69d2e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/4f57d608fdc63b18923ec5252e8e3a671c5980e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/672ddba6d0130dddca9c020ee4c03337d3a393cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/a0505b1188716e28598f60a7ae77d16657963a6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/bc09ce1f41d265baeb35f560280b0a82353cac3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/e447204745c890d7d677a2144d4768c90438952d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/0a94bbea22ea92ea8fc69a2cbd063a144c4f7c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/3921fcff0dbad3f2b6cf77f79aa14f2719d9cd22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/3c4741302b1cff99b3c02d9cbb2bf702d72e1ecd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/465b7a1263d19b2469a29d394d262ada1629b590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/63916c3084678ba54ecacbe3eca8c9a4dfbdb028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/dc5a49e7444eea25da8471024189592ec26e3be7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/30c5d71cd6dc1492d0b6d1e1b6a8de7c60baa67e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/6908060271a18abedcfaa430e9376bccc6e2f982.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/849e8804d38dbe3ba902b3d520977641a68bc1be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/89956dac04a965061e772564cfa2356ee780c830.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/95278843253e7207175087dd8c3a705282e9d15d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/feeb50c13257f71906e08b583cc2237be6c2bfc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/00a169736f8e31147b55d90fa855785e15ae977c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/02953b4b9242fdf0b837b54c6129d5b12ada6dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/ab04ed47e5a7111fc79c4936012792107f60477c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/ab913c4cee3d04d70e7d244a41b1b3b3d60f2cb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/e4368f7211886b0b0538599b2bca91cf0fad7d25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/fdbe8e841b60f2133a74e0fa505e2be634159f4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/1f12bff5bdd696c47ff4ce284bdfb72fe41481fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/5f6f7ab4d2f81064a47f0f901f36d866be41ae98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/6c5b600b0060b2e0694a5bc2b4350dd044f99727.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/a0edcc0dad8befbac0d22a2f598a4606750183b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/b4bc81f9a8fc1d728bd92118d7c065831db60241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/d3edf1d337ccc979283e61118dfe0af27739c5cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/06b635cd0347fd76fdfc74fb13215e217a444b59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/2ed6b3cc2be9ee1ff07a33695e9258793f7bed7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/3599c42b0ff82471f6e25fbd1d032b6ac133b4b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/4712db56eac10a7b38295dfced09d05fdaca8d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/777361bb59a53d4a095e18ad60e10b657d193ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/eb46e318d386a207a9c7d6dc1cae1906878ec851.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/0faa04e5aa34ac21957af1063c46114978ae9b11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/23c1bd3483f5b3cfa01bd11bb1490b4b7790958c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/480c101f4a3b038d92129764e5e1352aaf58b9f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/9fe12848322baaab48823396cf529ba6b8288d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/bd58c43ef8d8cbb4039dd0e1f693de97a0add737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/da975f07967b5e7214d1f46cb20a472420c25100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/12de8469ffe5b601ea22f8b1868b11da94b493b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/17f37f9bb7c8316fb48f9d07b96b13bfb8e70aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/7d04107a95f060716ac2b4938fd7252f56bcfec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/833669b900b7d9ffeffb197037dbe4b057a6bb3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/a22872568c4dd0ebf064e34b4919ebd4e74fc74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/fa820c6d9ecf5371b61325bc2453026ae121f02e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/49a43b536e7b4f38368db3e246a78bb9271a5b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/5d5b17eef4e7e258ee8d129eaee90e15fe2c5b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/91e6d9dfe9a90864c8bc2ee12d677a77f264b046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/b9e09e62cdcaee2995e27aa320bc3bba69a8280a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/c5e7bc7ee84693c0a227fbc5f7b807fa1e7140c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/cf48142730248248c6bfe2b807fa419ba2cd8467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/18258bb1030cdc9ce310ce9c54d5f568e8184456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/581df66f8131da327bfd6999268e1c0f1ee2157f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/699dcc8a9dd830f66b9fdb8b05baec453c3fa0b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/81b520dee4ec07790894e3ef04afe2b5b7aefc54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/b4d00fcbdf622b70bbe48aa6efa231c2a0a498d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/ee4bd4666d6088491ff2bed941439eda200ea949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/1e18903e9c1d65e2d56b7c5c5f09a37a9956a577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/32896fd9769839bfbf2273b2d59c8632a53a5c79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/33cb1dab3b0a0c27a8b7e0f01c9dc59478f2ead4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/926f687b16ce27d82bc720ed86ca09ddcd2b429d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/e16e5c384855b4affd50dcb0ad228c8d4d0e0d02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/f000128b4bd900ac587c3d24283fdca90daf6574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/6085af0ad13321883c60c755b45904cffbf5efe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/71d6404e2a74bb68ad83ecccbc88a89ea1eea947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/7d764a3655bb6a4fbe1c9d28785599ad1ab49e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/b5b14007917abb90d65490a589d065e1d2e3ebba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/c616e1cd7f760a6c8a4ef917de867bcc88557701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/f7bb8e5d0ee5ac710b6f01898b8246b634bfdbfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/0468398a1ac2bcbd651da14e536db9da1867a737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/4396974dffd36e9cc9eacbcaac880eb580fb8c53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/525c9a2f3bd02e44c8668101306bac91a285bbaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/807c294320340a3bbc0260c743bfdb71aed06b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/94ccfb994b71fdd94e2632293321055a49333ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/df9e9ad76f427542321299a155f557a4c5820ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/3d8657a1d3bcff35b530f9444f431e8354ee8304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/976754ebf13532efff6ba3cac13e74cf2fccd957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/bc72e1952dab31a6d47ffc32840599a074687baf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/c0bfd5bfe2e6df023d8cc4b58d563127cb8365de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/d6c63d6e6e3df1032d2345d4bad1dfa19ed0cc3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/f93c174a8bf0f61c36bca87e2e94ba8f85d662cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/093b564099130e22aae935befbba493ad5a2a611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/40c27be24b6486f10e343312ce0ed8224bf4b56a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/47598336b2e82842c80c67c694d19775d7ec7fba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/7bb2378863d87f4b18c4ee6c9c26ac5bbd046f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/9962714e94c01d87f20d78857becabbbad8ea66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/ea9b0728d1a360eb7ef6a65808dbde7ef9bd9fe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/0f80ea5ae845ab0320d17d447d2ef61b939ccf34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/31f943c60b509fd8d3d506a9c833530d6b466856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/4819502414831d28e671a8f067ab77c1a092756a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/ab508b6b33de79ddece8035e4653852c0a879853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/cba5175d1ce250e5e9a0e6dfd2165ab721c88d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/d071d95007feca6f48341300747ba48761d9a53f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/1a7f4babe8fd18715c8a2bba349abc7cb786beef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/4315e5f4074d6f0cf17ec68a64908645c718bf6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/736808ac20f27f017a7b6ad4cc7f0847bc83d3cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/9360fb198e786098ade4884955024050f3e1c708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/a0e6dc860eb4618ccab5273bdbcd5518388d2305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/b6b735954d8bd4fdb18473701f5a3e6e4b3d2079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/1a303e084a7148c71e86c5b0de7731537db5bdb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/1fb0a3e5496124ab30b63ebfd3b6758905823df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/771c9e6d611d84a23b6e644cdd102ed28a99114e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/d7b122f987cc3ed1a86c576a0af89f2f003b5e48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/da8ccd04d2a89afd1926b42244c170c978d38e19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/edebbe5bca50babb2d35ab90c805e63654547673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/51e77119a11232d1193551c28df0ef4e6cb36254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/78e7c3cee5845eea21768fb004bce927c6694777.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/7ed2d264b18d829cd688777f0f02a391950f4107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/ba9e522c850830c3c892b93272147117949a35c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/bb4bdc754256aa33f413f7add2755ddabd3f3b09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/ee16ce4a53d99281c49f1e30bfed45120a9cfeff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/1cefd1fdc0f6a6b09531f144b9e40700da62ead5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/49bf275c44c7acf1e64a1d11cddc1010be04a1a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/7ab9f96cccc16c1996b0e91018f9e3f54476741b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/900ca2cc3864761917ef502ee847b98722e53729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/916e11750e6cc1aa88a5ab3ba28665fe59841367.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/9ac8c81032559f4dcad59456824de9a3d2102901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/5ad21ae6bc499a27f8580702017525ccf270dede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/652edac14ace3355b66ace029b3bad2c200b6edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/76703937075fc344aee87453d21697f1f123c776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/b44c9515da94b30daca24b306eb8bfada9d70cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/dddc761034e111c1be4e432ac149cc967665eff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/fc94e3fe0b520f59ad74a10e48a9f7d141a80e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/5fa88c498fc368da74130be1f1798c6c597b5d22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/86aedc55b8e98222b5a9e299071d708d56cf2994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/8aa6e47e4d151ae134025de161e723412269815f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/c580eca8ff5b8c4bdc7a01dfa3c1cb24b3bf89eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/e1bbd4848c7b0ba7ef786d05498f4501a5c68889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/fbb403d0ecf723f000823159f50532643eb76e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/0724ddda879c906f3c4a76644a8f9d4fc8adb109.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/1d376fbe5a3b9a74f2deaf3659d399605236b7aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/74835fa7a513a9d0bab9e2a0eba1728f1e38feba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/dba12f99ee067774e475f8005f18a75b493715cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/e322313c52804fb6f53aa5dee946ee56256250f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/ffb6be58a15f32b90a4591e5a51107c1f8f31f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/11f990498c79c13cf82f6b110bdddb388e7ff73c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/42d1065868bdef7288f347610522f616b42afe30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/8abe1c19ad5429694f0826dbcb5bbaf34efe9b88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/8f77d69a84619d8f8d7f40a24fd7c2a39ca045c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/c70080fedd4665b75f38ae5fd4a23335dfa8ddf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/d5c747e670db9fb2e9f79fb4a990f06aa025612c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/5fcb8b4182c461f3883fa56c1c1445f77771a243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/6668ca52a601256677f9b899e55383467ac0f199.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/a535b10972bcaa1d227b726fbc8f5473aadd4fec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/f6ed724a573a89250ce4c2d745b47efc21f99b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/f722c5d9a29a12c3dc3d5b9553b3bc280bbc18d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/fb47559e6f1da7023e215dc638b43b7189ae4281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/33836ef569455d9b19644aadf1ce3f37ee75cded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/3983f95baf1e8ccf78891b8d63925b49620e4188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/4ab9edc8c1a1f2028b4154d1bbb955bde7ca838f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/5f35c12d6bbf96a180552c6109323a0ca50d4800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/7db1ac24a3282ad413833446a2a9b655423a8344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/ddfbb35185f4d2326432213c4d56ae15ef2d4588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/14d0a4e4676514678ff62255cf0ac3038fd7dedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/204977c53a591d0219bfb00f979272c2288fca36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/3b28bf40ae1dc519f1f055df4a92147f152432a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/5c80932449efa93b73719973ba30dc0cb77c9086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/913dbab83f30b7f904917788aec91336ecaf76cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/d29cb2dffaf8a1869c9285519837a2c56faffab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/01e4b610fbf7acbc53b54e2bb8ed78053e560227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/55cd2e285389fdcba58d734db70f88631cd8cb5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/580581b832b4255ae3c7f4947c6c5842cd47d6a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/7c893e9435e4729398a54d448dc23fc60c888841.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/b5ef28835482f6d62fceb54c57f1cfb00a3e0801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/ff0f7f93740295ed27107159fe5668ade1e44fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/53cbc4ecf91f75a04fc5a9531d27b7717fff0e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/5a53916972a8df971c32194848e6bfff116c88ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/73dc20800f4ca6871f9e20f4468133a4c0b02208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/9a68e2908398be21559360eabc4fef8a489ef136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/a9a8472678be06d13e12c4ae261cc8b7152a627b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/f2b4b2652d6438f1ef0f071b273f5c5adcdba6ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/66e1253b9b37c592247d5d4d1a19388f51838e9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/768de7fa77a26c0e55bc7c96863c701122d398ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/b230fafc5d17dbffc7a1205545f39374ee4a6254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/f305274bcdf701e102ecb053fb8d2270229ab637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/f329a82fa38ec92b065d8604f5162210595f16d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/f534b6d17df2af5d4f8fc05e66554711b3af06c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/048c721cf3227a832a8fcc0421236beeefff0c9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/102184e76b79ba7afdcd425dbf5c868de2264282.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/350e4d7853f897ae1e45888ccc8977df49923232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/7e65cdf9c38636374f3a980f79f4dd383a261cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/b034da7d9d00fcd70c80ad78239e4d72f01d6463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/ce8c39b34451f7ea7817eb2f025d0a626af251fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/3a8de441499e82df6cc60d923ca3190c34449c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/55d1cc27bb11e01f40913f6a51bd9d1a6bd6ba83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/731035126fecec2f7d5606d88e040c94a6dc8e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/ad9867fd44cea5dc1ea51d10488b72dbb781802f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/da7b9688d57f88ed8e5027922abca21a4c7eb6bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/ee86614624d2b32acd9ad2dbd1b3f6325dd210d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/12abbd7a1b37ebc7052d39606b8c839467621b8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/6b304e2965c4848c31fa460140ab3da20e8d40c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/7506dd3873a7adfd2eed59de869dbfb7583e6c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/78bb15559ec0321b5b32d674b9b05c6a16f23a4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/84e753bf5e6a43276dd9f6edd3ff30b1beaf1943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/e8055aeb9a160c550ae276770e0f302b3f594e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/15477ddb8e658b6e43e42ad6e22a6cd07ea8e690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/21c17b6260a324e7868afbeba6b5aee6c32ae033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/691ceb94c19965aa7456d8161dcff454e8203861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/d5bc3103a4b88730ed9f307b783a92c9dbab8ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/e854904885af9bdcdc2362a8ae32af5ff0f3d1cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/ef80da6c4dead789332369623f59b32cdc647876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/273e1778e97e1f4f6d3ae7460606857471c9b67b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/8a31916e0d6130191a6e850fa174f07d6c0b7a3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/8f25f6f4b73aac337c468cf6cd4a4c74ee19e447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/b2cea8d500f49379a7d113168d3bd6f201d8ced9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/fca6730b02e5f5b6b68fe3cb3710f58c239e399d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/fcbab6ba1247ed72b7f9400e4b2fa3bf537d484b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/1a1135a0d6bb625bef792089f4fe9bce87a9fe0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/8b2c7bc3112639828bfd5e5f99d1007c5406faad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/91e50fb969d103118eca60cbb8686b3026d5ce32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/94fbef1661bbf533d40ade46d17c91e6f497f52a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/9685d58cdfc83462ed62adf42b94d6e79d8ffb32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/c4957fb18abeeff7cafdb91e8a03d7ed64911812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/054e5a45a28f95afcb37d5e73347bcb3a0420554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/209313d136bbe5921bba1497fdb99a9be88e5dfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/8567eb1f74c992ddc4904ebfa4197b1d1c0bdca4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/cbb0485fd6dadfc8228a2783ed1346916d08637c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/cd00724261ef353f40049302940d06ad73c5be3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/e918aba0c028e521ac3e196544136b6afb529f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/2e2247029a033969eddbac4b3b04ef077b9e5c04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/5660318f624f8f7cc153385232707e3a3f1a4918.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/582b6edb43f7049bb1000e5a8be8d0defcf5c4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/7a01b95141efe113a5d86bb7e9e2c138af8e481b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/9eb85181b97485d393be7e20ba410090a6aa97c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/fc0da1f0c1e9923823e7b1caf2e35218f4fffa96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/46de18be7b027375cf1b80a153df78fa0b8aa308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/8583257544bd8d3de7a4a497cd9fe093d824f4e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/968d3f7f6a272cf555945be3975664d6a8ce2ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/c471160bec1748321e9604daafd6c8fe39411c1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/c7d6b51b7ebe2e04cf0d0f2edd9edb5433c5b8a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/f0e8908b51bfa30bcff4470ffc7c99cb317eb9bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/0df2f5076b3e6f4be258d37521a7ff3cd77ab604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/1b9d2f7419389b09f766d5fdeab57f5133973b52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/6d7f7145b46c2933caad5b2f87c1aa03d3e4545e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/7543813657dd39ab500035683159aad6fd6b3f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/d6066ad71830c29f8b49d9ad00d5e0842aaf5d15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/f2ed8d782c8712ecf9268bce634c855dce2402a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/1c56b70ff9fdd324eca0c926820873995d8e4c44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/29ca31799dba8a9612b25e21d6bd1877ed6e2326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/2a38071dd19a456abc6f27d3bf705319ab164b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/3d24d7a3f998c797d68effa9f970059c212165f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/44ff1d3a9edfd185c3004c34883c987dcf3dd010.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/bcd5d29f8126a0f90f77c02c55cc0fa6b1889cde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/11aa44c5000b8ad6437d47951c330320c016faaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/20b57845e445476efc37d523cc4f118be198e122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/5516042a3f8565267466d6b33420fb693dce05df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/6625b968370c59a79c3b30482a74eb2a02b8a8a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/b833659b83f03e77a216cb1a761ab7362cb9c2ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/d36b7d3943a7662d6e0135429095c40bde8d9021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/3e25631803a8732343cb440b62c7c44233e8b428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/878f87bc52048ae46bdf3a4fc3ad2018c20f1d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/969a3ddf8e4a2fdb6c19ee127e653fe90fcf7d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/ae11f920836917866871abc7a5db95ebb696dd53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/d9cc3b91e9fd0c8a9c67eeeecd42854435436351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/e3895e46c7d154529e983be8ee6451d2a94bca9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/0c109ef0c868c196be28bc073664fabe42737bc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/60eaf8e940522c6926f69614f9c7f8084b451c54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/6efbd2cd7543c6a7ca7a8a64428f496173da375a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/d3d15c7f977f145e9604dc192e8fb62e34d67729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/e1912942707f056576de72785656cbad4e870fba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/f58d4b14b1b5f0ec7d06187ea06a70e72df29bb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/0fa74fea3adf833559fda896e35a6eb4d408e64c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/346a97a61401f469cf1c5e072ed779fd8933ab24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/6ed1205d0aaaf734f13c6e36159044326b918df3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/70abeff3def95525a901bcb7c00ea6735aef37f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/e262506b3ed67c4602f34eed9018bb85ee90e958.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/f57197b11a18b187a630b191a125561fd7a90fec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/2d08058041bed0aae0a2e46c684fab725adc36a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/41b10a620a5126edb6ea225357828c631cb37f4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/a897607d7641722b1ec773be25171ddcb6743b13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/ad8779a0dc21b8aedf606e0363bd281236127e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/ef3c84226b252a540dd3f757fef16bbeb7bc50c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/f63bf0c34de9f6fb0ea9b6a3f1a4e8bd7bc3fe6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/308bdeae63fd1273d401e27cc3fe5669e824fcc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/56d1354243d7de60a56442247e543410e26645cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/73f1275dced8fd3350f26c6a5d164fd1beb38779.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/a7aece9e3d6046938eff9f1282efcb96150dd68b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/e84335001c2b89604f405371812a8cc8723e0a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/f9be5311a99800a736857fee69368796d6bbafbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/101a79f61692b1c112458e66205c10939829a53b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/22bfe875800203062b866d0de26b3aee8a31cadc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/44c1a68caaf7b87758ea286516db534d4ffd1cd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/7338159bf291d6d7bd775b1b7b61186c0f77b517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/8351e81292d9c5394b2a08909ca893bc3574ddef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/b833497397d43f087697ca8751af833a530053ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/1305485e0a78bcd505c9d9552059001a8d65463d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/4708e1b94bfe938e1f84bc408f9476253738f6de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/4be29cd764e5dcbab7e001da2741f44e0dc673e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/582bdbfbd4caaa788d98f178f093dce352d25bad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/7a31715a2bfed9abbd758acf841806b19df4428c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/f85d83cde50f0958d8ed90fee398e62c06a53753.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/816946d6a6760b6812f4500b2a3820811602f079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/83101920e6f9417d618ebd6e65c98c82b526899a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/b9bc9f3c4c70f50a154a60dd4dd6b6240469d2ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/e2f1f0cb5432128232a45dac6e5595676c7ed9f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/f3b7085168c90ed7106b9dffc4244a0bc9171633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/f5499d70047e6828a4311da100a10d415106e9c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/11ac022e3241a030cb3d068abcfd5c36d33722cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/8b72c389ec9478bc7ab6df347c8ef35f2bd6ee99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/c6e665e55e2d5a06101801fd9423a14b98c0f794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/d44575deae55e9e0d48e7e4cca3987457ea995c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/e3d76981b2ace2c194747c98d536fa4e2d5c95f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/f01e5105b545df31f2ace7ad8b4611f035641759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/0815d728584be93e3fc2f0051ca0e71acd7713fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/0c52d82082ffb4130469221ce024b4f7d4ce0fc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/6460d8141102bea4135c2e67b2544741dbf45c2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/92079a7b99a33a180c75bf3c3495c89550ac365c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/b352e6ce005be05515719dc1a295456784e39e01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/d8ac79f26e74d7afd54c727e48e9387bd88c31db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/0e26cae1e32eba8f38e9ec090862e4f6f795d869.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/1d1201c7fb9c2f67c8d0e2f3876ba168d98ef5d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/3062a0bc97c63e39932a03ac4cb8aef1bc992d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/945e8e1ea4e8d3f0ce31f9112dacca033bfed5e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/e50eda86dd73d035d2ce6d301e1cad1a31c3b50d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/f6eac8cacb4bc11ef98e4fef4d2603a359358b9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/0ac298f030c6259af18b2eb4d3358587fc2edd08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/3584eb56a17f918e77a2dd3451d25d43be0aa8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/5ed6b70b594d6da5cf66f9214d9b44d23ac8e900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/7a0841ca101ecbd8095c76dd03034ab59b43b407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/b1c1b426f4b6aea9a694f1af9b5486c22e4975fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/eb371a2698d003e4973292ebbd188ee5becbcba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/235f5ee02ad11ad2bc2560fc6e9496358d2f1dfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/2aeb1630ad79d8f013b0383fc7fc96aab7a85d4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/652a88939c52290ca44cd80d54d99a9e21dcb631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/6e82e65de6ceb24f47066a6c7d2763cffa7c7832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/b6aef7fe520f6f2bb589431c4cda0a2b4d143569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/cd6758a82b809f6470ffa8f91c8240d03bf6de0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/3b02932fb2577a06ce60b783087946f0818e3091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/731bbf4b72b2e581c80c8e9170be76b3b21fb444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/7390589875bebf717889a8ee51c1d9e2edf3d7b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/74682ada8812ddc22747a63b7829250d0f9bf3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/7b3baf8b75fff86ff6293475d231ec5c8bfb35f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/d20c8cacc1f3fb94250cc3fb66a8cb30f7f4bccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/1a2cc21e808a22504bb5eea20229ec5e006696cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/3f284a1df1c2e95a6c62628f695b3d3a9ccc8ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/56737ec9a5a4c81a2aa4de1e8e0a3f1980a5d898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/584158db6b31795e4f75474b770f035ba356dbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/680ca6abc979200e8c976ac75fb14573c60ea703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/9e098aa5bab6c8967abee493bfd393ef0063e15d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/58fd6eb33d2394c5325188ad7f59ffd3febc27ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/786b60e15aaf9b5adecba9d7b0f788b93aaa6ec8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/7b9a6c5b35a078605af7ba59c270dec76221cd5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/a28fcf9b3a1b9155764c96706a137e4705f377dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/d63a163ecf9f9a9dfc399a20516ec4985ed90e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/f5a2e05b006f7e4ea6b11692b853039f5a51afb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/1d1962b38bd4923d9ae04ba37565c0d9ce926115.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/3d479b9cb5fd76a0daca076fad311753a925c261.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/4583c72fedb7e91275666f78d4ba3741f3074f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/5fde24dd5dfbf12eded9f6fbb17fb4559a3973ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/8713d2fca39d2a6d02b063112da04354bbc5d596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/9316704063848379834d7eebfb793c5b7ccfbefb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/20d494f65fd4b565c50cd7ab2c7e79aa6ffeb79f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/2fa9f291e2eb2d2c5c72792b52a05e49d9439b5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/342a9ad836093078f90859d6f3dc94e9696bfc17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/93cb6ca81e0826580b567fd026d92e0d3173e18a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/d87987d902b04b5e71053da5708a2248f5ad9c73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/e098c0b417b3c544718e30bf200abd7faf909b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/05d0acb8d1cb98e408cbd23c9f873fa478dd71f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/6b74d57926c66b8d52f7aadb8eb08ecbfa8e95fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/78855661f25268378707879e8752c1bc5233845e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/821e5e166455639f4c5f2d464885b5aa007d6b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/9ea901d9076d8971c0274001a18272c5628991d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/f10df8ed4461c27498e2a2ad599758e79c767b8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/090108a7be1e2885cb16c9a787ebffdf301530a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/119767b98238c715abb2c6e793ff8237f320b9ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/473946e8bdb60b26067fcdb4bec92eb2568505af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/4dcd16826f605ed61e0f01d1e958bead359bc9b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/8739e626c9e3c6311d35df48896e7ce9d08ab83e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/aa9cec3ca14baff2a96d802d0acd743f78629bb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/2b818472250d9355189ac8aa4d2b76973a36930f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/6593ba19c82a97953b08aaaecc79feaeb521a580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/709de75c126f7787fedcd7eba431061b94c68ea5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/72f08b6f7970bb44f79d359d585bbd97f92e3a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/ca3e5afec9f965a0627edc804b2aea2c6eae85db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/f17430a637131250ed07ebba54e1aaeedb97cda5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/4455b7233be103b8d4f510e3dfbe162433f2622d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/625eb4ff00eba5de4bf815ae9bc761708630fbcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/748df3024d3aad093a934ba067a8dc5352e35006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/a7e548716088149a8a3d0d565bc1c204c40e959d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/b8cf0683a5b0524aaa4647ae1f812f9018a7085a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/c47ad8fac5b5dde0b3da33f1001bb45642996057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/5e8423ac4e0909f404361b5fbc0fdaf57bdacb87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/63eb75041f15ee339a9e6a124c209dbc59689b9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/6d4afc5771fc837bdfe70ffa9640e01a0cce5486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/84b8843c9f78cf45b0935edd09465991949c4324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/de0735c8260dda125cc196e892622cf93803ad53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/fcd0a95dc6b931abab81682af1d30871cd3f31a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/339d55b90cd6b9cea08e2621b5b2a02730d471c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/601a4669eae79df75e4b7235423921474aee3cdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/a200ec380456b018ff943a6e0393d8bc4caf20fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/e69e2dee188efaed1788160fbb6f8f5d91fc9a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/e932f712baa6632f1663af0df705d4dfaaad125e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/f7538207b3e68c198d47c81fc9e9af747da01546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/054498c67a141fdd0cfe7ac2beb582739970147c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/4dc125234f6630257d82bf337a3ccae5d2b716b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/6d5a8f8d47caf7fd8ae51c67389e8fc5f1370b06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/842634b3af003ad6a2db5f38395565d28047caf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/aab4979a8b9d4c1bfe649d627791d96739ff8a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/ec9d49ca98fbc1e4d34b13ba9dc024b7b0f06c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/145dec8ded2a0072b2c0a2499561d0098f36ed33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/77fd1bda12d8407987076994244f557b5b1637f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/841b2ccc67ae589ae46e1594c6ad81e453afeaa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/85df8ee4e11ffbc15b2a222426d68af1f0566edf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/c804c0b4080b261eb1faf49940ec0afa7e348dd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/f70ea9f14e15403b3a8c1eb15f636badc8001766.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/3fd5ce16c0a7800bccb4414e8fd970c04a360419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/922d6635a1d560ebb9fb94cfba19779556d89e65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/95fe59187da23f1dd6f6e8191b2944d3cbc268c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/b08b58aa18fb052bc30967a10d178c496504c930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/da1bee4a11219bfcc151d587fad6f62a76d1f300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/e091c21aa2356d0cfaeec6d656d656b61735ff95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/16d18045203d6dcd69fc3f375e22ed1522a46bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/29706c214e07b783addbcad79f37d814955e6d0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/5562f9818e51d8293ce14295b0b6e84bb67a6b69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/7f2b300ff07b58bbc625a872b19713c36ee5661e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/ae9cac7425a30974efecb4d75db20f846f44d3b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/b2133ca442d6a746801f8142ad1637aaea3548c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/1ee59665fbcec030b8ced9f7d76a8b169b87e13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/228f86a40aa167262f717fc52870811f25bb37b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/23f28da77fd0def3901489d7b8ab05e1c4a8496b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/8ad089e72780968653be94f184903e5d3435c74e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/ac7d6d10f635aeb318b1a606bdca6a264dd347f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/fa700483c677994b08cced59d750fa85ee039b6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/141b50d0c94587fb9e505c05ea2e11b87301b4c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/521508711443b52867f5bf3a1b56a7406f186737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/80b9ec50913738cecb56a969e8278a142eddd368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/89fa417b841fc15f8e475c5427050ff0dda2f32c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/b5d794897fad22d88ce361c13deedd3c18a68107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/eee955183618feddbfa1e39ef68f81df7c44ce9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/33373bfea5e4bfe5c1e0051cd54295283cb743f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/3916aa353ef539d7dc05415759479dbaf319d44c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/922ed27b21da54591b1cbdabb07ecedf582a3a5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/ac0de1910ccf4f02543a9ffbed84c10eb9c866dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/d069a1bcd26589614167abb5bb503e97dd66c89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/d8ae2626eee7df39e94ffc92f93ce00b9aad6aa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/7043a15bfa4775a7e9de55d1dbcedb3a10b2fde9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/8e6bbad084fbbfe49f0cb6b3bc117208baf2e9d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/989848ec154c1d28d022d670b55061445b34d400.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/9b135e23bab7befb37687e7c50919f3359f81d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/da4520cf065dc394ba44961a0855fe70e36495ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/e16542c4a5050c84fc0ca6a6f0cce0a365b5d0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/05528d239c02fc7ab9cb95937c7621ea35e34092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/6df5d6efcc65c89b27d2d7152b1a9ab753243523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/7f7f54b1a88c177872b5d4d4f4962aaeba0ff7d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/9eb2bb8a719b85f3c958f255c4d436e486f51e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/dafed5900f9c5ad14a11c16620087464839f7103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/eee96c4d8565f306761cb588e332d4dae64d09d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/08bd1e84d25becb879a9ac6645d464a4923688f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/3b08c3ad236516e8e7d58254b96cea52276ec107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/6bdee85ab48febb71ee0d7fbe4a48ca048184d7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/874d64464c02c74176cc6a00b4fd9b33b1bc2666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/8e8af5918b853d8a6d90c68b780b66d00f92a0f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/c5c14d8efe25a8b26dc646d63178ea0ef115ade7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/08fa86ca8de381cc542c5b79f74422c84ce18f92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/2f1c70ca7b1159a4d2422e7b4af5e3ae731b288d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/ae50105426e9a86826c0a862cd3a2c7e3e43b804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/afdf6275e1df112683b59d13a2f0d1db4d582e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/b81717d5a1c9ba8f941c38569d307164d8304418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/c98bb924cab392c5a009a2420b9998a557d22482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/76dc829f0cd8ba27d82e842416d7721f20213ea7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/819e38436e9724e585dcd437efe67f40c9e25f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/92a7f9a88e4c72a627a6cda68f177f473f07aa5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/aa4554fca867267e5d73f5ba34d4fba54c6a1f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/bb0bc51551925600535c60a484356bfdf08aa7e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/e03faf1ca3bc93d2707dbcd60c75bafbd3cd187c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/074a0672b93b3f4653ef26ed94d3004fc01d07e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/65e7f18a2d1c451f9dcb381919d3c169f21d0b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/aaf2c1fcff40e739a7b64242fbd100608fc78d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/affe18bdf95be59d2c79c6fdcb4d70216a87cbff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/d17d71fe35891d9553e86da5758511062f818522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/e9b8b89f3003fad98f7184c5430d59ba84c9b2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/79fa199915ad01926b290f7c6ea7afb201541ae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/7cf186bd66fe01d86302dc34ed7ccb3d541ef5d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/9a785d28ca3e5ff8c7e2d4eb8a8061683f416e6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/ad5e08dce73215c2bfc07e904e08702074f409e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/c7833904d3a592bef75a4583832101969fcc89a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/f558ae316ef1f49c6e757afea769a0b968ced0e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/22e0a74e112f87e54755e075022d52452b560d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/373ec66986b75aae95b00b2a0d09d40ee1c24234.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/46db730b2490f45c937feb25fe5985a0bc5daa6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/4e6da717bef1c1e6e520d52af5463c0affbcb957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/644304b4304b796060c02c44e4ec840f0d63b413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/7d6143e68ffd18e5f09bf25ea4e9cacf93f8407f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/19830ab685f3fba961788bc2aaff7711b45a3bed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/293bf4602c97fc09224d773e54a0d1f39b14e796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/44118d1f939ff1f9b6c1b9e34aa578a6eb373797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/52490b1ca5c684cd018cf36f6f6a5ef2070361db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/a48404adb84b295e3c83edcb59cc0903d0dd4b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/f11d87fac677adccb0a4957fa95eccc8df269584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/0399387afbf337c2ce38c15d4f841d859600f92a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/1f038041efb12d687c8ed05ff9266aad86c4c71b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/29ae80c449cb7750feb2bac54deb540035c18d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/4934873a27d8d3aa7130628a9105d791ac565615.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/4f461ead7a7381c40c9059b714a62a9a7c41b5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/f3789c7fe0a9c656723e3aec6643059ebb874b22.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2020-34/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2020-34/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2020-34/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2021-17.yaml b/configs/cccc/dedupe-url/CC-MAIN-2021-17.yaml new file mode 100644 index 00000000..bc46e4f3 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2021-17.yaml @@ -0,0 +1,256 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/0f033336b6fc23696baae74b8b083b0e3a13c8fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056325.1/warc/0dedcf45b255b94a9fcf76f616107a16cfd22da7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056325.1/warc/17b4d198b30d37e7d6402adfb362f479495186f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056325.1/warc/4677324dfef4453db078c95dfcfdd3b62f6ddbec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056325.1/warc/bd43e7175db8f80e22ec34f7c42d20dc152b2e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056869.3/warc/0998a555969bde86e91ae565edab949b9823f878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056869.3/warc/0ad813539a52dd19fce1f180ae738b1d20d01b0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056869.3/warc/2738325a432dc034da24193a9a7ebb6c7fe079db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056869.3/warc/59e10d0ad4656d34b57a26494e301e51b114be1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056869.3/warc/b5eb1a478893818e6fa892c0b26a182229b0b0eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/01996b2fc1b09865cb97ad1164890e485a747a94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/33cb33fbbf878e6645b07be6962e1d16fdb1a797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/4fc0895593ab39526c9b87ad4128366de07ab033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/7c38d532579f218f1d06d8cb6b1de3bf31ccb89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/87b55d7ad74dfd62646f06cff6d9ce7e0142e6bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/f169f85f4174b3fe312c4c1a0d6f08d052883a83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057476.6/warc/10b1b39456bef1fd6f5496af94fcdab0a57b62a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057476.6/warc/3eb50431c42ad1978862868b1be8b8df4ac44328.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057476.6/warc/4c558e5bc39f442bdd057b089988b8f17a0a1580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057476.6/warc/8da2c02aaa03d53d255068c1b96d6e445ae613ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/12ea9fc4e88a4e4c469f20169ed30fbe81f7e213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/1f1288359a9d8bce4f9e1634189fa293c07ae6d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/5f260ebd1b4a720cdbc48171bc937c64afbdc6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/9b0d3c360b030aa544ac6998999088e368538db2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/a3fc4a871639e1494e93e6a29c0637f1df806a49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/bf326d478578ca39fd1975fa964d130b4e378232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060603.10/warc/59587a0559a07cb01542593ea990f594f08d56a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060603.10/warc/afd0a94a20553424461f05ec646e14d8df3261d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060603.10/warc/bbb3521a0c65a072b63e04e0069871debc004b18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060603.10/warc/bcddba0a9a20182cdc37f389f9c91a7ce92c0291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060603.10/warc/cca66f26e681bbf22c33a4173057501536a0d68a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060927.2/warc/159a325a8c45dc7599c618e87810388c677dcb54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060927.2/warc/78a8e0c2bf754bd1037cc364956bf8f0a87c5163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060927.2/warc/7fed51bc6cae0c71e61602663d6caa84a472e0ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060927.2/warc/9ff5bf437ade8e694208cac487616a416ebda481.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060927.2/warc/e356cd82c38e18b60df8dab502fd4d4cf8392f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061562.11/warc/a39e84984e477c823830f51f6a2619dd7402df13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061562.11/warc/d76e09a753867d8ec1e3dcad1a2cfff52ecbdd8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061562.11/warc/e3940dfbc72dc358c98aa22ac0c5796438e2a8a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061820.19/warc/770c6610e54098ed36e0d50686b941e88acb7822.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061820.19/warc/7870a387529f2a73d1db6559d5666dc0b1e75193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061820.19/warc/b33104fe1bc4973dc142d819f061f7f501375b3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038062492.5/warc/0b1cdcbae5ac0ea43a13049bbc0fa2c418bfd0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038062492.5/warc/71a96ebf61568b7c4bb24d5ef8dd5bded55861df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038062492.5/warc/7f92316c30a5efc091c171ae9195378046f516fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038062492.5/warc/b92f2d0e235db263ce00d731c42e34589e917b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038062492.5/warc/f79e82f7e1fad2139171a7345d72db3f3c7584a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064520.8/warc/2c99b8c9cc0a914222be09e6fc43cf09f49c291c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064520.8/warc/3230904facdfb1ffa5bc9c95c973e476d6ec0d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064520.8/warc/5ee9646d620d694382ed704361c3d57a22595a58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064520.8/warc/7f7b4b4953d42fd2e38e34d101dfec29983b3904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064520.8/warc/e05650d9f83e9d6f0a303ee281ae265695ea34ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064898.14/warc/3dded0fd6da541dec10b4c26aee5057eb2419e3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064898.14/warc/55585c2b8d63172cd0e01d9537f30e79a3489cdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064898.14/warc/ae53b45c6efa31bd3e69cece2bccae9571acb824.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064898.14/warc/d3e1133bb4fba4459bc57ce80a0ae4813d7830e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065492.15/warc/1952bfde997257b61472aa071eed52af6d43698d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065492.15/warc/463ac94664ebdd19a955280080b26ce2a5ab8269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065492.15/warc/5cdc16aef6ebc6a89aaf4909db0b77103fcf59fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065492.15/warc/94a8d6af021ba2a73a891585fe4b5df6349d552f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065492.15/warc/b035d2f3b21e952fcacb8e8709fea5c3d4836fc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065903.7/warc/6dd280d5faa231330a8be89fc34b5902e550ac22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065903.7/warc/6ed357ecbe0768304724bf800c262376c5fd3cbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065903.7/warc/ad381203d8395ff03c96adfa3a869dcef4ac432e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065903.7/warc/c8407b66708b32340a8933a1682f1f1d9e29886d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065903.7/warc/db665ec69cb44d5cb6249159c3a4866c1208dbbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066568.16/warc/2d3ca5bb42c5e235b47532b2728e135584df9344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066568.16/warc/8b5de058b8569fab2db23792afcd8794248d8127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066568.16/warc/d87792c3f577b7d2b984d9f9543d0dda11a181b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066613.21/warc/0c780f20bdbfa1901dae02f97a9e3d282fcc362b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066613.21/warc/8f9344b32f452ecb963ec96d01ea3b3c456af925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066613.21/warc/c0ca9362438d376213bedb922c386a2ba73c6fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066613.21/warc/e63c0ecfc409482bae8007d19fba686b69662c83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066613.21/warc/f30834d3e017a5da7b8e41763b8f7aff80d21e2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066981.0/warc/bf373e247b2890fec5124f687b8a90c41e3c6439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066981.0/warc/d2897d8a04884ccd8ceabec2d42b7849fb05e0c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038067400.24/warc/06846d5e9d1c77c23b54af1b67b8d2cf1c73ad37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038067400.24/warc/747a56dd51309b2566f5ccd866687fc138d8f7c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038067400.24/warc/b3414a4af610993b6cadfec05f0ccb5f87d7741a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038067400.24/warc/dd21edd7a7ca4f2b6b60065662257e11528c182d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069133.25/warc/2f85dd311c7388f4c10d9ea0db257e5847833c17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069133.25/warc/58172f0d332107ff295bff56446af4a4263035e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069133.25/warc/827a2f5acf50c5ca4e5846f4cfc67596cc55b442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069133.25/warc/e17047a65d2a76a67a60d845a7cd9760be959613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069267.22/warc/1601dcdcf452253caa1fe87b379fe19c33a23940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069267.22/warc/347428678a0814831f3e250def28dd85fa5621f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069267.22/warc/8c79cd8049af2c21df8b6a43d263667914691a5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069267.22/warc/9116b7a258603d7ac99f24a73d097612ed37ae17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069267.22/warc/921b292d299117db5d60016d44ed113e10ba2a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038071212.27/warc/7e5be3e025e54eca49c97cb1761b3397480f5eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038071212.27/warc/96ebb90bdd830f6fbda144d70af67b15e30e81c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038071212.27/warc/b9d1ff657ee8ee41ce3a3fa5222721645d49192e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038071212.27/warc/da1db5a0233bd55d37b785c1077d485b1b51f3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038071212.27/warc/f00008d4e5c64095f2658cfe22e87f7a6cd184a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072082.26/warc/0b50447a3681f388b83f69f27d86a05781af45ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072082.26/warc/105b2565a4071cc98c5c691f99bcb72a0c0d82b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072082.26/warc/2706a31e2fcb7aad5566dd42a01f4460d9e3d98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072082.26/warc/93eaa93c48ff1b3f30fff560730e974557cc18c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072082.26/warc/b7e584f7b525ab776f34636734b3a65d75d8a28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072175.30/warc/4e01ff5dbdec3e171e8a72cc8f83158f82bb83c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072175.30/warc/5396416877d882467d4fbe610b81f78f1e6c16bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072175.30/warc/c1805541045c4b9892d8f816ba1ba57257a776f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072180.33/warc/14de4de4c2aea0c21b49c8e770c63e4dcac705e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072180.33/warc/1d749d84c34e226d56e09d469af558a1653580b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072180.33/warc/7b5432181549190534eb929af56043961c35f13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072180.33/warc/82ade6ae4e0f6c2bf0e7809e94b5329a58e9cdee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072366.31/warc/1a82cd633cfd01884e3f219400342c7f2683a63b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072366.31/warc/31fa1be4636acfd080e47e1869158209df965335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072366.31/warc/54a12ab4562b7e77c88b54c8d26d205dea4de742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072366.31/warc/c1bbf750c644ec29ea9125686af003455138b52f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072366.31/warc/dd6ba5746040275d70358500106b9d19eb0e5e89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038073437.35/warc/56631af906f83069a5c704569483b09a0b2e9170.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038073437.35/warc/73e09a135c5e37f349c9e04a3e9a26c94c26a5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038073437.35/warc/e502b15c70386479d5f2935d3e44bbb8b4e5534b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038073437.35/warc/e506a73ac13399ccac948d7eb101b84946ce2185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038074941.13/warc/34bffc5811b94e52531afc012b8b375a1e569fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038074941.13/warc/5983f758bc9655905b9c72b396345e9397f011d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038074941.13/warc/5ee03232a3767487367efb50a68ba92aecfbc139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038074941.13/warc/852cb7571b23063fd051637d599c31fdf8ce1964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038075074.29/warc/0200b1d6044bba261573eb62ceb642ae41aea901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038075074.29/warc/43e56335646c8a5cf3f9873aafd9218fde1ad78f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038075074.29/warc/6257db78e5225a091f85e92dbd4e25181f35cc25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038075074.29/warc/cf5ae2dc33a602f04ec86b334cd03bf57b969f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038075074.29/warc/ef85bcae3a1860a79d155da479a885dff4d14d08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076454.41/warc/4c1fdae8ed5e553e1798795de07f97808736cb54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076454.41/warc/8af2af399a67ec970937c0e7e1776ec9280bbc1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076454.41/warc/e306ab6c1e316589f31d1745484dbdf53d931708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076819.36/warc/6834eb19f5e4919c244306e8fbc8071766b1d104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076819.36/warc/7272508c56a6bdf8edcfc51e19f38a2f4977938c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076819.36/warc/77f6a32c24005ee21d18cc4916a16576301c863f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077336.28/warc/98b6ec5f0662d9e1df90d4325d0b95419fa2f0bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077336.28/warc/affe7ecc0c42a92ee977633e6a906f97a31056f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077336.28/warc/d45bb4938213faee612de0b832655da2babbe0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077336.28/warc/d4a44e708ba2180479016ff67230661b62488d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077810.20/warc/031c97a71f8b0a00a0c2f03908eb3a6d3dc5505a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077810.20/warc/4e603b53ac577dba1b31d3a4ff6ec434c5c676cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077810.20/warc/69ac852ec605a2ed85b2ad4ac4f9b1f6530ee7cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077810.20/warc/74ce3e2ae44cf6c727f2441270006e6dfc59892b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077810.20/warc/e1cfc359e858300eeee5a0500df373bf9681ae81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077818.23/warc/3d0eba0cf1fbd20d9ec0f34a6cfa28bc7a21c50d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077818.23/warc/4ff617527d59be509cd0283bcde0fa07626437c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077818.23/warc/52b20d90372f9494c6a5adde7e2b9af05f3ab04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077818.23/warc/67879fa064a684561693c3f2013a68afad0c40fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/17541b3c9195723a586880bd346a7fd42130cd2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/41868fea8af5364cebcf0a9d166935f425a1a57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/6090897b158b9b51baf6e2f96e00e7ac1efb6df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/6d4da11219b13521e80b616f6cf03f63f43857e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/804f101c9920a8ef08ad72af3ec8da510fd1bfa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/c448525408d778987801c7788ff97f9ea6e14000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/67b01548099e3c6a17e63c1d1942faeb2c82286d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/7472a1f6a4344e0153fef201b504e058c84afc26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/81a8a3b9b8c9b52f7a65c280d6a3e57a20232d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/b3882491e7356faffd0fd482e591c8b3e5fac414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/c844a55f4ce18a6c4f028cdc5f6992890e4b0e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/dd5e61c42b8556983f3200967d58a57e1ffecc3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078900.34/warc/4546c57f682b683624653926b21ddac69b9d6f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078900.34/warc/7fd26bee868a85b94cfb5a11a46d23fecc11d49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078900.34/warc/d543128f89dd12f1a4d3b26a88c214c1c9a5c3d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038082988.39/warc/0bb7cf567925f59937385f968c4535c3e70ff624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038082988.39/warc/288e7cf70edfe652145f2af29415a18d817a238d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038082988.39/warc/689bcde1d4ef7227abee57ba7fae043847f70b87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038082988.39/warc/c779fa235cd8da232a63e0fcf91d52974f73df74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038083007.51/warc/40a3fac9a1ebee1c63f2185f5e952ca7c5fa934f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038083007.51/warc/577c5bf605c40e422c8c7a811438e0921e4ffc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038083007.51/warc/8cb5cf9ecbc90a151f949016e387264a0451c042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038083007.51/warc/9d3bb29908672c2ffe8a3e225ffced76a75ea1f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084601.32/warc/0eab38ec93d1e29bc41996b14e48d21a103d52e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084601.32/warc/325c3fef8fa021f3d956ec7bd5fcb849c426aa78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084601.32/warc/9ccb819a5e7411cc3f1ba4b875253841b88d7457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084601.32/warc/cae3bb2d8305ec2e5ed2aa1b2c9877c085d17da2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084765.46/warc/0ba75e4b21bd0ea1c8e5c654cdef7fc74cf79bcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084765.46/warc/5a840cc08c2802f4dfc19acd36f272b135f044eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084765.46/warc/b5784d269e7a9bbe5cf8cf7982d4bdfcc2741b10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084765.46/warc/dd1d992b51c5a7c1385b70f71325d11c16e35198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084765.46/warc/e3cd9b93e665211543fc7f9a26e7ed381d8a093b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038085599.55/warc/049ded267e95afddf1f91daf02e3da717a6204b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038085599.55/warc/21c56060f0844f74f31bd63ecc6c5f2e482e0540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038085599.55/warc/a98aa70d2059ea9bac408f3b6a7746df88abc24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038085599.55/warc/ec63c69d30dc761f3ca4cd40810884ae069926fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038085599.55/warc/fb1942ac9f29997a403b50fdd98685e2fc3771d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/220d1db90abdd9cf538eb8f2f80b619a52efb88c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/3c9c2315110ce1e073639e995f96fd6cf8584aad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/6867c73e3debf4583475ee1f64db3fc42a35c7a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/c3e35939b667053e76b749ed633a0c96c3cc4a68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/cb964905bf52db3eda4413fc59ac9b636c1f1f7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/ede265485ac5d2039276d6a556279e6a1e409ef2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088245.37/warc/22864d66611a29f10a7659447b62e769810fa88a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088245.37/warc/5887c9586c8b7ec99ce2997647c1c475cc37c4fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088245.37/warc/75a3c87abb542e653fcaf86276af689759b043ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088245.37/warc/9d1690c77b9ad95aa72bfcfe268a40b056f479ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088245.37/warc/fa75b7d734794fbaf05388deb674c88b593f327f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/315e511a81a248f49836dfb003df6b964d45ce0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/49e54b988bb97d5ea0c641d7cd158564bd1cd301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/5514d13bb8174d68fe10cd964b0f6e488c20bad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/621b885cce31c8c22f68f78c79e925eea599b9f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/bc5049618ec4d04c00eee4217d1382ec1ce2396d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/c06fd460eb5362eb07c205e0c23dec2b5986d019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088471.40/warc/4902763365d1831dba702121ffa7729588f93d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088471.40/warc/49fb7ce601e29fbe0ffb69659f50044a58fcc65c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088471.40/warc/7794801770889c2db5e57fa01ffcf93793a57f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088471.40/warc/c964c4ded3f2d06b492d4c6a17de2a362f7d6283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088471.40/warc/cb06767509d58c9f88f6f68b2cc89d975ae594bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088731.42/warc/712c47c7cecbdcd466cdc79049719b9892a6b619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088731.42/warc/75166437fa89badfa8f79affc263603915cf3670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088731.42/warc/9df0ecf326419b5686186040905876e4eef2f501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088731.42/warc/d268795e33b74d3cc8f6acdb8590b863ab737818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/169ac96eabac86f97587d97bc2921f12a8c3c272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1ea3eed47092334d78c4a9d73b69d34fc9855cea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/205c4b3d55bca0f528b2d37e2703acc08d786356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/26aa35e9fd9e8c55e08b10486a82fb041c12e738.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/2a78b6564a479114be6bd6210ca5e460c0f7557d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/2d67f063f668ed73f1788f97c3125d2700327fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/334986b40abd2b89c8aa94da34cba6eeede90ad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/4845985e0d64bb06fa4b35932586dec5ef9a097f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/6123a796da321dc97af14ab4e1b4f6b3fdf6697a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/679d43473b463d80c97fbece98f4c90feb0e2ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/69a606f7cad41076f83b076dea76fa774f33ca5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/77accecbe890f780d054af8dadfdb4cb522d31b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/7930d029bd9d2051b7d682c6a44104c8e47dc529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/7b4348ead441bd4a04fe19265ce07c7001bd7826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/942dd5421c7adf4b24f73ed29cf1653a5100eadf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/981cd7e1d64ea220578b97123df8d6e4fd799d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/b0892dc2a852d66a3ba974f7ed5e1d7a058b729f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/b542f1932046e081383d015e8f35b4e8c89efb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/b57a299af4bf02e57c11279a640d81ee4ba93ea5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/c38fc5421684e5117aeedec1e85fe0a92d99921f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/c807e63ac006507bb8f81d3245f4d7b825c5a5e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/ccdcd1031a5e70b16b096706e7ca4246a4ff313a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/cf5f75f78306355f8b33727c6b6b826cc5601bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/d3a2a155b21663b7b093dd6be37a451da2de3de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/d45380191a19770285139c8a06ab912fc7363e1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/e1ca1d9c43ee0e9ca69ee2a9ea5b6b05d110a189.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/e343ef94e741c230f921da0e25bc1b29de24a072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/f666541b6878f882fae0c276d5e836ab9929ff28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/fc16fad391c56ac4a4023246a285b7def1504418.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2021-17/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2021-17/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2021-17/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2021-39.yaml b/configs/cccc/dedupe-url/CC-MAIN-2021-39.yaml new file mode 100644 index 00000000..f1c9ebca --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2021-39.yaml @@ -0,0 +1,706 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/0065ccb72a390dbf3af24b25bcaf1342b8e91539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/012aa3658aa001de083a9c9f0cd8ac5d36197e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/07d2a5e2e46b0d1c919f4a9ee3ae1144aacf0f8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/11b86319368e0a1759eafe2defc46622637acbe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/11d823aaf342e048db34b56f6f345da37306e3b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/15c91eab2b9e79d48732d883aa5df92577cfc74c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053493.41/warc/5bb7d95087d106c093a0884e43228310b11ae227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053493.41/warc/a76d3c6e2bc574e939c313529269793a87766a82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053493.41/warc/d7143d94130e2bfe46c230a7e445cef85dbd3b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053493.41/warc/eaee95be3f589e116c74666e2d919c99faec7c73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053493.41/warc/fe21e74128603be87bbd29e6e0f4a9280bb9aa09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053657.29/warc/143e8b3a414606ad0d0f544f3865fbe544ea9a35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053657.29/warc/66cbc06eb20292650d1cb8c8b293503cf2d183e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053657.29/warc/6876c0c439917d476e6fe64a1058d2c3e890156f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053657.29/warc/af120117aa0fa756b8a61ca2b1343f2c96d41355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053657.29/warc/fc179c96c6ae6f739fafb0c7e3f483939f33ac3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053717.37/warc/2ea6e46af69514c106d2847c9b94daeb4f11472e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053717.37/warc/6b0202c0bbc3077adf3b21a14a5fa283c1297088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053717.37/warc/6c29212dd73ec621a4ffeae4d4931556034a3b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053717.37/warc/8e5abe6fb55a796468706f5201e61ba5da85e529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053759.24/warc/33b83f8277ee5e6003079f896599608224c733ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053759.24/warc/50c53405dca569db69847ed2180722d9878b7d2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053759.24/warc/c0c7d516cf656480d7c694950410522c13bde9fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053759.24/warc/f606bfc0747d51d5ee9a512959f4c6b901d22139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/22753cd342998568bb68a4c064a30f3cf574e2e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/5aad12c43c6eaf2945a703cb9fcf6e41a2463f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/6ab195ff82593511fa93d673c2c0874f0568d945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/7bae6e8df50274e4c963fb9c22a552e5718ae4e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/9088d6703928cc681aab57c41684d2e1bc509b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/be9e03c96dc767e4309f3d07ebbc1d3f5b4fbec8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/f68dd6aff4cc629f2590b614da3bb1117a3c7838.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780054023.35/warc/785a44ae4893aa2c28515c8084e615e73dfbb6ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780054023.35/warc/903bb1200188a24eb5fdbabad032a783262f098e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780054023.35/warc/a73b396c56b7e9fc3f9ea860741e8e79c6dfef89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780054023.35/warc/bf9d629fe4f6a8f71f6de5f9af7851d0ff2ab071.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780054023.35/warc/e3261892089b4925c84c9911abc779eb7f4f1c16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055601.25/warc/088ca9093e43fa5066576fd1c8527eda7480ee3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055601.25/warc/716e922ef1029aac83fbc5626ae9eaf9eac64113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055601.25/warc/b73d04a32ede695ded0703be7269d9a5c263a131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055601.25/warc/bdbfb3c93bf80a523b418475ae4611e95b261ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055601.25/warc/f91ef1d47f9fa4735d25f4342f0cfc50d667a3bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055632.65/warc/048381c9518a08ef9968e427ca7d96c5cb30edb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055632.65/warc/1a0f36f4e16b33d73ef7542d2fd131ca34a53769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055632.65/warc/46b141e30b2942121a1a62737bd8f0ecff06efd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055632.65/warc/f67548e79db98f06b3bd43f7e2cca12d43d18a20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055645.75/warc/884b6645d964b2452a4058d83d212f34ded79176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055645.75/warc/c23094bcc2a2ada3308959709593c22fb343e8b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055645.75/warc/e2148634b49946d671b07dd7152109845e680f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055645.75/warc/f539c239be3221be47d651f10d485372da835559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055645.75/warc/f65b3d996843607c155ccd4df1048543cf4b4e6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/1c8d0f040b3c1d3117de3462ecd321bf89c63544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/623494cc30242aab97241ae4ef6996500f65257a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/6a8c6f91706f19b84ce22f4eae9160f3b9ddf9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/86c037f404f781a914a866892c31d4f866aff551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/acd41d5f175bf8f089e5e130e1b650f2b21a0e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/c8c794fc37eea85e3c8a87aaa64baf3b242da388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/1f20a515b9b23cbc2ef6671caacc08583fb5afcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/31b66cb94c5c785ba0a4aa47693f3755b03a37cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/3aa8e43f7aff2446303fb41bf7102d14934b59bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/62b35cb4f7a5d79f93de8c8bc43b8cf3b544df5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/6a3fc1f37f91dde36e38009a0ed644057703310e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/c6862f1db3259317114f92836e7f400b7df3f0b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/1091783f8356c9a4b7bce7b29ecf5726312e9787.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/38cf10b84f528cf2357337000e267e1edb7d3a6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/5fca4c34e4307184a3bff8c62b6a28e930cbbf8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/a80e7c21744fa7eed6db15f0b7e951ed48d5238a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/f0d2808f0e1d7a6981e4c58319acfc0f146fd2f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/fb33ee47ebb5b37fd00a226228232d374021bce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056120.36/warc/4c83a12417c7322130d4a0019032ad018990e4b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056120.36/warc/58dd78f7d6af0301189cafebe0886cfe2f5e5a7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056120.36/warc/854228a6ab7808c63a1dbae424210bd859e92785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056120.36/warc/a6b426181f74f037847bceef9c398962e8883ca6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056120.36/warc/d3cc7a194a45b8c101132ce5cfe5fc908a549ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/34a465d2e37160fac8c5860bb1fce091f14f57fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/46f37939fab7c82cf6f0f3c63d94527a28c95d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/5fa5343552587dbbd7e8b75457fd3617d52eba59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/6825c81a5f8370cdae10e3066cee4f7e0a0c3074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/895df609835b4d17f2a8225321b7a5d54c7b3c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/ed195e6e43864e0edfe1eb9d7c71b66051f6042f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056348.59/warc/33f3c22507a5cc80861b3a2bd25fd4ec993a32a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056348.59/warc/676a7837879d7db3f25c1057daf1e4a67e8902ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056348.59/warc/a26094accb38f26a7b37a120359faffcabd74e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056348.59/warc/e501b207d1e9965680b00a1c708abcbf490e3170.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/033c94ff592fb70c73a09ce7022b00255e0fc292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/42461d8be9c49ceff1aae710107881ab5e870f2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/9d8fbb8dcaf0e77fb43565f6eed6c6155a622dc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/a87a188c1e95d7185c71be2377a9a48c055385c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/e4ec19c2d9a0d66471bec48aa11e347e4e9835aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/f18ff93d94bea6075d06207be59210a998925cca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056476.66/warc/e96fe5eb66541411b4489a9bce4479b1550f115e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056476.66/warc/f43ca2b5c4c3b28f11d556cc48a267a1c9bacc98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056476.66/warc/ff1d04423c4bb9339610d45caa4b0aa2bf7d07c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/19a5797c42cab5dba2d38607bc4cddc613b709df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/7cf71deed280d05e4cf8367286c18032a5addd94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/816fc58e81f60e883c1651cb902ee33fd601426d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/ae397e50827795a4463bbfb1d794748887765ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/e50f7a48168e064f1ce9ccfdf32c4651c3ab131f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/fbff349f06e6f80e7006f15e47f32014aa9a51f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056572.96/warc/1aaa618b0088b1860a2d6991f40be936d89b6740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056572.96/warc/5ea72087d2f45fd28ebca2ad8ecd1f78022ff1d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056572.96/warc/ca4b2bb7885155f72ad4af09ec5acb3b21e21344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056572.96/warc/d8a234a64c97523b19b6c87603b2a33b513c418e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056572.96/warc/e9cf9b36c67b204ed077b14e6c64fc01c400158c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/137ee321676df71e5fc21092b264e0ef9eb66d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/560c60cbf1163d7767e8f2047a500eb069e3db02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/a640782d8e22a45e81b52a66ae5d4df4f311eae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/a7aab91d5901443746637e9a56d07c0a4f2ebeb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/ae285d73222d0495e37376a31d2d6555649aaaed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/b2c1da30e0d52e3d7a37593ba4cad98a75756c69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/c43f5505b779de24e9b6edbac84c43c3139154ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/277f59dd5bb52bee949e94415f5e043274ef2985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/32faffb719817695bfaf0394e94c2fc49a07274b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/37d6b6a0832d24ee382d59eb72a05eb6aff35d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/b7f38eead74fe0723f562b9b4192fd591c3392e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/f23624e999601c9b8d6c324caba8020130296439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/f55d0e20bdddbb29c326b4be05459a317e2b8a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/01507d98322ac0808333f9bb31f1408164d89e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/2cde5cf072696e67ca0eb86c38d888a89996da73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/39562b4856a6e28bf04e92d699c109c5d29d4207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/5b39d13f278699f165c53491e605d857a26199b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/8c3b64491340278f539de6c786bd0c0d283cb10b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/ded0a6083b1757e4a6629a8bd8c6e9d02fbc29b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/02fa21a0a24916cf563344d763bd5ba3e66cfc7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/1ff7b50699cd777abf9cd92a628b04e9cc66d06a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/4f1090a8651ab68555e3cba5407dec1a745185bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/af0cb98763a27f61f9812a1ef9183988d39a479e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/d344c8583867d361dd9d110c7677748f504b75b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/f092671e5278422be4bad62b75938b5469222ab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/0c5d5d67f07f0fef78245607992453fa2765f478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/2a20bc1d8c04021ad766e2aa24dabdeeff604dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/3a776ed34584c7c5b8f525a3826a7225ea397898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/6648d431741d9a88855ff37d6d7f2631a5e73cbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/6db9688145a85b227f2129a584cd7845866a99ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/d4cfd6f52bc6ea498378aa9a0b90d56ba08d5085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/0b36c8dd7e9a6c577c66c8c11ad5a3cf3b4b63ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/160dbf90501cd8dfc347fc058257d2b8c3c437c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/2d5393d2b256e676287aef976ee62b2f9c8da359.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/3a15cdf249c029df188595d66fdee56ff43e6620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/644c26b422ffc71cb59f0cff4c774f3dc62b737a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/758e2603f4abe0d17903b6e84107de7079f9d5dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056892.13/warc/645046139b9db7ee86c7059ae8cd254686c3dacd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056892.13/warc/6ed2e20c7ed9c12fb42353f1e0ba71bde26d9270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056892.13/warc/cca8afe3e8409f651dd964dd5e425edd2aa5a751.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056900.32/warc/0ce95e4ea3b4d5da6f37cf200b2bb46bdd0a86ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056900.32/warc/0e04c3b746080dfb579f3550440363b69c9074e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056900.32/warc/193ac12f21c15adc7faa7737664eadb745662711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056900.32/warc/66cf1a1677388cb2ef7ee23f3838dfa5ee55ac00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056900.32/warc/faaace150bcd81d79b088a0dde93a81ffcecfff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/2ead3f59653bc1a3bd3ec8715f73f362d95fc985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/469be77177ffc614b4f728d74e5ff1f315c68038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/931fd456341e5fe89cfe3f5d5a6a2681e37b0cd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/b3a2ca24350504750770bfc7cb62897b308b7e87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/d73271c60ef2625fea8e128497b9ca59e8ea8dc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/e7e2f60389d9226ab850e3615fb5998287b2cb32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/0ceb6ec4821a7ca00e55c65f772e885dfdf2e47a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/71885a5aab9e9db9b57e803fc7554ca2037fb0bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/76f6c11077362bc47b90c015b0ffc49f47df41f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/978c5e77d49a248a185c65a04508111e5d2558b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/ac1345297641d296a0dc0192d53054cee68f630e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/b864598a1d47f3ac9be225558c8ee833e27ea3c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/025c90f0c4c814d45453c547e0b1ff670e09b3b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/1d57b7150643061be552ab7198cb6e59fe6c3f45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/2d748c87fe88da1fd09e2b0ecefb0b5b0b4f0fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/659f1032bd6b019c7f7c8643c517792a637cef66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/b24e865d2e4b26d5077b3db654742f4247362dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/c9b91ab06b714ad6ffbabe7a68f399034e284d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/d0b8cef1483a32fd08ed5c6b278d2551403c63e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/42227e996caea6211afaa1e9beb3cb68518ed1ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/53547ec774b3d48571240d4651c93282705cfd4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/57580a6c8abebb621b6010bbd29f59b8e36cd00e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/8e1755cd966830c2387b291b357227c434912914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/a3a25fd0b2d426be031ef3c47c11e69326d896b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/d76b00f5128c618a03a1bcce40da33f396be1a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/d77dd6f2b994c08613b82d69c6d4e5d1a6c3e81d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/223e89e267fec0c5f13bfaba0f89f3a779d5cabc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/24914bf74850703cee59b7805d53489196934d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/499e6c40856c8e6497de94c8dda79ea8a9cc5f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/b5389ec49d299ae3d885e3297a6d95486f0cedc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/b8add31ea09d114648745a6c404ed67acc661752.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/eab7fd6c50bf605d9c6dff91f0cba2871ee6b3b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/1b370d22c08007eb8fe0c953195ae8b7b488a662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/3ea2e1a5fd311edb1fa0cf2c1f02bd6691566a5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/4dc6331c942bed8c971253ade4acbe7f6b64a547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/60c12efa4fb57960878fec51b995e7499ef41209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/b7716eb79be7a897c151abd1e7c71d5990e573f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/d6e824bc844486deda253e48add4f3afd9307439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/030bc2ae06051960fa21ac3082ddc3687356fa29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/060544ebb51b089e0d78e7914b0212ab1c9bfa61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/66ae21c0445c0654408f604dd0f6d43427012f30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/9ccdfe0e74681b147f514fc7519f58d79e23ff05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/a62ae462073a636bdcbe6f53ed01ff6947bfb1a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/b7277df76fa75e977fbd0d8398a816e4b49173c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/0cedb9f24582aa336a121dc233c878760ab0373c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/21620ab863d5d825cef62e6cf5968bdb112d8763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/2d8bb90f2b13c8c60e119dc334e49231a1999609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/37ef722f8ebc00fdbcadd208b0fdf94fa96309d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/5a67718915936d372aa65d7378f410e463e39272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/ad583aad57e65300cd094269a76698d2bfa38280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/e8fbc8e5626fafb363de60053d8c6239706a32d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/0d09386c4d66fb11a6430fffb32491ceb287480c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/34a58d03b84f617be0ba36bd96244c784ebf3712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/3a8c6a332d171efe4c16a5cbfe42c1279eb62569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/6029fb7ac8af4abdda60a0cd068b39e1c2a86c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/b123078895e4184cd07d4465220ec85c86738fd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/de7ee39eee9af96277dd754bdb1887cab0ceae58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/df0aae2a95de147f3b4b05b40410c294e1622d1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/1566b6926882e3f81a2f67bd32d831fa196c9904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/1a8b5a5e99cb705e4d5ea4bb578aa4d16ade8875.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/3855c18fc3770406ca161b0d6727a0ebd5622920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/72938884eb75d16b2b88b1458166fea2551a27c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/b633cb8bd920b9f1eb1fb4e3bb61ec4f34be3d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/e09d5bfb3c01fe95f166a045190830992c96fc4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/4831ee6364acab1867090c2da5c75469e0ccbcff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/4feeb5f2e992da2a3da53e8a06e64d0cf5896c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/762482761147c403d623245f07fb1445e063a69a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/8f004df40a81f2e0e4f83636cdb81e7d56517574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/c390c524122be0528f30922afb0e9b815aa87fa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/d72928a21b14a57d120c4d2867fea3f847ee034d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/06a81a0c5379c27ee807f6e1e9c51d29ac4cb631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/9efdcd00adc833728a240fd3d93cdf6d65f5f3ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/e54d939810c07ca7dc968e931258470ff07935d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/e9bb0442b458a9a2f120a2241ffb9160df3e3f29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/ef7f3760d079eb9e9c36372cd93c5b736020da67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/fb3424f137d292b6c84d13fbfefa9c2906342cdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/155f10af77ccb9696215ae335695b697af98a6cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/2f077e7d0c873c30fd2be28d869b7619de0c6e8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/5e45331c467ccdf9310d9d8f545e8bef26eeaaf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/7d1eefd3522c458c6b8aa3e1fbdee99ceb71bdbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/8932b440802acae163f403b2de7e9ea67bf1e953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/c68fe0af6d20ed575f1d6d540b348339ede70375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/ccc83d1e8936dd806e942481000c0e87157c560c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/0ae6efef97c82f8271c5487b7de95851b46b8b18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/3b7a02cc7e82ba07f8b7228ec20fddd677ced9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/46b3a183a5ada7fa6c7110220cef7a28751cf264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/719efffb5bee916a34d4df5817f0f5e753c98c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/94fcfe4f708f5fc23b50af06bce28679b4a4fd21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/f65fc8618f3d04c60979edaabcd887487fabc313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/fd76b95f686e9ce20e7d29ab48ba6bd29e5debe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/11ab8dc826d388b05bacf33ef6e6be6e30f3537b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/1b72b561ff237ba1b7e2b4ab3096ca91e6f6b11c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/53efbabbbd706d8ccb9607d5f6954c522cbf41bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/68967577f4d9aa63369e293f954d0f6c272f1b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/7fb0ce42ca666e6b9a832712ac8503ab476c5f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/b96a760691960a1ef160026d06d515c0cba1abe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/0e7c351dbc9fb108a56e804b1011cd24d6ed11d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/1b7a29bac20c3e8fc81fb3f323e8847c8195afe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/c83e163e2717b22fc95c7b619539157f9cbad815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/cec0614ec62fdf9cbeeac73c8c09ddea00777164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/cff758c481933efb61b69c257a354e045eee4aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/de298a82471587c47f13dadcd8c7a6ec4c064763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/02619bdd266bbe846ccec54aaa63c37e83b13ca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/3cc097ef7a1c77a9737c56ab07a697d38706643c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/485ac0c59e8fedffd548b0063389985d6038bb0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/a401065f12a5aff8e13aa1dafe907226ab1a35b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/b2cf58472b837b58f3c90ddaf39d2f0d9c09868d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/ddf9019780beebbb6ee00391e2577c98a593e821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/35bd774d75f4095cb0eab464729a14c0e21ca56f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/57d87fdd6c2e59de17ddaa1559037ccd5a1dd30f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/8b311f07c4a92a2948d0a17d8f4a0a86f3296492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/9c4e166918b6bab62a8186320cd02501f385f6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/e0c7c349d05a119643931e9881ff9b00277506ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/e68e99ebce39087b516b9326a7b49e25598de124.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/fbe9ff5679c036f4b6b49fb5c184ddba120a7862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/1c8c35ca706ac1c866cbe0a0494861ab074a1f04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/28c78fb8e1ab72d36baa0957f97958f97e1454c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/495115edd523fd1480a2a184a78f5442639b1d54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/647d7b9f659061ca10a1175df76600944cb5080f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/bf5736625babb2d6ff571c90e4053708ab0294e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/ccd9b496b813973560d641641eef4f955eae364e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/eb5f9e40ec0159db2c9e54c8bb2c641a0889f855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/4812bd089bc769d16173842c33a5be0269f484cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/7f3738361a68d783bde2fd0401bf509ef2736a61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/b20dce194e5167f56710377278e3b7875cd71568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/c133ae50f9d06c176929f8870c9ccb71578eb121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/df7b309d95eacdce485edb9a7a00f66541506f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/f14af009f9d1b20ae543e4547445eda12488a69e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/3f3694dc8ccdd91ccf2ab80ef82c6b390d832742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/7c926da75878b4a59afd6cc8fc26dcc08c9871cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/c2180e491af23bc12fb590933c30986df1c72614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/cd6eca58a0657426c5138bb6f2b2ef9ae19771dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/e55c53a5bc8c510a5601ef2a40244c1ebd7a1948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/f7e20552ec817e7d9a2ffc2d0d159d3b2c83f1fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/1d6b67e3038222076139f548599173d3c165261e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/56fc9ac383972edf67fd72455bfd3a22800834b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/98232745c922583464d241dbce524ce044537004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/ccdfdb83529587c54e580ae27f7d642ab9c9bba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/d1adc382f37200168fb5ecf407b006d28397d6bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/e71c44110b4ec3b83fd7c9490e7f93883a1f13c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/171c3d7306f44e749ba18fc3458a06089f47514c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/1bcd9f8ec727d6faca57b1513df5245c5e16b8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/1ce6a0f93e9251f36519b88c73fe7651c53d0ec6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/2a92fb56b3ec3d1b1c6febabb7e2fd5cf0d20157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/d80a508d8405a89c31a4868a91aa31143c6c5e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/d865c3b5bd85c4929917a24d9cafe74115643af6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/ff0cb907affb9e97af7fdd03e2b231ae7a87c4ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/11d61c1111e0511877962b946c719fbce0bb1529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/201fc97d0a303860c205034e7fb2462c00f2ebd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/2aade69fb436e082c208704f86f21b953813b0ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/38fb1ca4df16bacd7887db09d1b1c226048fea42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/8c899d356a5563cf7ff093f6be5c20509ac99c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/9228d8183d73d416fc850e0f37753981f5e1bdb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/cfaf2d979368ac961ccf90642effb90d71843d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/0b92fc01eb7e828732c2a62d46e14da7d965c492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/51eb6f477265999c36bfa31e7b18ee13729192a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/8a3757494f0a1073943a37e4260754482bb74188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/91b2363695f5b9b93b5153a11040fcb338238df0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/b33e07ea8999a32effb9d0f028915c418bfe3382.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/cd4f43abfb5bb5bc6a396da06f43b885ec43230b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/25e6d64f91b56d01749c5775663bec0501cdc5d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/2a15a457aefecd3cdfb167eff1f97a73e4df86d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/2a2eb7a386eaa3451bf26b2cd4b7137a230790bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/c352a5acb18702ffcdd12f689ae38ef959007f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/d283d533b3cec59247d046ac2368a0b9cbca956f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/e1cf497839e0005da39a72f20a0f3c12414c887a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/37c11f1ab25a32e7f3c11191aab6e798680fd7c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/97eb3882171e8e936823f93bdb22c3015eac0059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/bbb85e9b48b0fd86af22326f148db20372c414f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/bc4807beae1b2d8c7b9e7eb7589881cbff5c9705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/eab4d7ed029e6dc66a9df4871bb19a17c9cbd7ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/f8d8e22a4ec4a4b9350e7ab4ecdd5bced163d9a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/33c42c835dd1e62831280919b01ff078c496cb93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/44bdc1d8773ca409ac38193895c53b787690f84f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/6a33c0211c2b0ad387ff290f1f105c8d869bd73f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/cb4a7231e0ad8271a41a56f1d9270c1a7f8d0c95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/d6574f9cc7460295c2c53367c237d281e042c17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/da4b97633d11085ae6f61c4543bcfd58aaf70840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/e263b3f096d183b8de692928c625e8042d680978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/0278ef4ed1cbf8d31fd77cac266a450a90d8f323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/457d3504817b007451ebf79d68fbbf69a7d786ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/6b8f7e43f51255e04e902b3a5c3bcf66b811bf59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/9059eabaa8475e3defa3335d07f81e01516b0747.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/92f7d059aa20c204c833b70d20efaa85efd12f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/ee9cab403471b03e3d66c6460a84750c5f96e212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/f283ce2de7a82d05f7c9b9db1488aab43a142a0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/15cd982fd330a86c4b15ed15658bc36fdb8af06c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/9b2ec072a1c375369005e3ecb55d3cae799efeb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/d2a3baf9208cdd8536e33c3cf742249fa538ff83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/d67f9048b5d51220df4fc42bb0f7f6dfc9632cd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/fc362e7a2debb0f06974f448a223b716fe9a1ebd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/fd801402ec3024481753d1ef29d1d49747c6497d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/301e141c504cd3cdd8592e7295399810a3ecf976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/6565c471fa977c32f1ae0e7ac6cee4d3ee7a2f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/6f8a5e18af9b4404d57b9d8cb9fd760656710833.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/7fc4abdec1262d5c35598f95697719111855b9e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/d201e1a7525ce6b00cebb21693a1cb8304073a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/d991bfba50eab205cbda585fe0996e38508bc190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/0083b84199e5bcfe71aa9e28c2ed24192958b28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/26e985953c2e258dfe3decd7ff7f6a28d9979df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/6e47a4e259d675c5a11c9c4d2d894dd3a96ce725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/75a12b2f86a18bbe852bd9809e53695a90d82636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/99a82461fde4e26f2af26caa6c44c88ee30e57d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/bc70cfc6b543857227725c793f5b4b1ef9bda8fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/02f328966789620d0b52342f39ed7bea6a108ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/22b1aa0e16ac28265926e756831747fcf8ea7c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/32b246f5823880f0398b892755c6fe720528ded7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/4786080d6cd8ab16bf62e43627da559b2a909d05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/69b25054c43df98743109040488fe0c7332d39c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/7d8cff8a9ecc76230f113e02915e052aac34878b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/f23617df70cf971120ffd64c6cd8a292cd9aee05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/001218e599ae192f4767dd13b52d6f45ab0e32e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/308f5b12f686152cb35901c89398f2b0e77a548c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/347033a7cc94806d2efb5502b0b702b3780b8e6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/553c726e67823358d8d63caa46156b1132299d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/725cf98f08fd701959449259b7831d59b25ae485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/8348b0088636ece83fe21fc4f5dc73d3a26d0889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/e5c7ec93acf81563b587a88863ca84de3d0899b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/2380b13c80e23591e565efc6c23280385035b165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/39c88000daa2917bcbe2dea9f971ceef0a744176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/53acdc52f0ca73639ec9d0715ad814a086cbdf96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/67fad2676cd3e3ff5d8277e8c8fef0f39345e1e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/6c56eb583a9deaeaf683347e44b5a696e78a9973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/c407d80c279c6e32ab284e8eaaeefcb92379627f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/064a657f30215ce724f0ada50f1e03ba8e558a56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/33526b32ad8cb7d8671ae0f5a0f1e5f6d603ec01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/3f5681f512e77308ab6001d2893077e5135c1eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/81895f05d4a1a04a81a2ca053d0e586c6550aa70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/ac72e1d586d142ee73144203aa86c297f13c669d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/b1a1a73eaa32387b220bc2a32cdbf9eace801f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/15ac1f1ed96a3ba1e8afaec4a0b4b2dd60b773cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/7b90330836b28fce59807bb879e7cd18615799d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/9274b1af5bfbb77f2bb195cf1e9060e830709b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/962582b765277fc505df875c422332a4c20aa875.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/a2f56ac57343009c26c2b5546cebc527bd1d4567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/a6010fb28b675f8b5e2cc94cf5d85127c9b91324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/0ad62451066833bce548bc008cd403e01cd508f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/22df456c89efcec643de5d6fdf6ff7cddd983148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/5bea1764197220a0de6ecbc939adb6be05224ffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/7e6dfbf357daeab5eae4dac14307ec559184452d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/9b2ee9903753f7c2bc3bc914afc42bb74f4fbbe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/a2b72a99cb49166dc07838160887eb2b0d1f505b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/ee34325523eff7ae90a143bcb2913205e5c42612.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/02feed2f9cf709ff5c895b929a26627ae32f4f44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/0c993a81a66e29d0d6a5b007baefb7ab9c4a405c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/2b16acc219cc28f48b350e0ab1857737aa9863ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/5b8d905792b2728700b9f32a96b892ff9d80c7ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/9744beb76947af76a8621cb11734703dcd0c4415.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/a0de38676905e504af1d4d675122dce2fe6b3297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/cf6359ef6afccc76d432afb0fd4336eb959bd5d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/03a11762521f6356025ce3e055ab2c4a5d2e4e58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/0989ec41bd8958e02e639c04c762502331ca405e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/3b55df9f71f2a6f929b778df507448cf1dc9bb16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/7f57def2f77e3d51e42b3a085addea53d27fa73c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/fa76ffe57f7b15ab52e708535c08d8d6b7505cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/fd3768c89d7a052444542f49b82e0158f9466c5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/2fe9caf2b9a5ae2c04256bf7ad57df3a0bbfd62e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/5abf88cb3baa975cb6c31564b2a5d5797ac7d3f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/73cfd74be9591875bc6cad55c1d523d7eab97901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/91ac07e879d1f7201b6c5305e86d3c2bd3349ebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/c6c9901ebdf9618106b6ad8e532dad2991878815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/f9c53be09e69b11de71e3ae2f965a3aede777abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/0975eba0095409539d87776167b0704d4a96028d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/81e1b2c30679278720650bc2f98530d536a654d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/838c58a145bdbdea548a42b3a1ce23c0d6fe0d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/c749bff53e1c59a96bb5f3abab6564574f0f0595.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/cb433f753c799604fbc10a1075937778d33e9d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/e42a3aad035d8fa50cfe08adc5dc188c20672098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/36bbd4b31880b13f009392d5c0de58c7c277e8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/47078f9a2e04e1a245ad889b1d41860058a7ef58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/8529f2e1ad3a1e8ac9a01ab0f017433ad196cf6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/9366f1a8f78e3219c3dd471fb6461f5d8eb95b07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/bfc1948176cad48f033274e9ab137e8423f8bff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/cd47fdf3c415571c5fd8cee4281d1bb8a7eb9cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/e53ccdfe8b680aa5d24c5764243369b2cbb7a4da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/0bf8dc93587e847ab708c268cb813908a7f84346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/431b4e6582781a7d3dd8549fd48e672ddcf4080b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/48e46fdac59b9306069297e4b357f1332d998460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/6f303061b616a8ecee5bf99541a803794fbc2591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/7d4ef5bc2aac81525f0178bb790e21a94dcb8d00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/8a8fb3dd7ffca6b1ef70781c3eaf142706c91b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/a21d87f03eaecd1828807c898379cd5c62125547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/2d3407766a28040a97235ab5c358034d6ab842e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/3138b251f9aa89b9da301d237a0de39fa645f188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/51edd95408abc192a769f6084101033b63328087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/a2e20c36e39cc7eb72cb3273c320d5c0536e24d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/a58a18fb3b99b7e7bef2fe6f6d18e7b748d0a7da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/d7492e9e330387741180d718d681be7cac5bcead.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/273eac31960fdd96ae2fc56cde81f7bef125c38d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/2b449777982fe6098a2e19c15861ef176c0cf034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/3984857fca0290799933bd3b0cc815b5d320a4ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/3b773f5badac4d42b123a3dd1e3add9c6ddb1de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/69980747bee40145b5391d4c04f456ab9f24d257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/92635e915fdc798585b82cd9d4dc098440448a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/0260d549c45146495424d868f20209b36c3b301f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/2b2acc7671826c6371b3899dff17c139201dc39a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/59effcc7c4da7b8a9530ebcb9663a5e1fb0e8d45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/6531f0a7fcc92910cf47799b0753413d1e8e96ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/87349d383569d286e3dde19505779de1b496a2c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/b6de8feb6178da53703e50982ff8f49320f19f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/0550f4ec04b9fec2efbc34096cf3d6961ceda300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/07eb01d0addbb1d05b78881025ecf68897073c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/18abf922922f2bf11aeffc3bd2a929d363682220.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/19ea7a4eb51ccf8bab08a18f53351e3326bf8154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/a0d29c6947c45d6f5f4e5bee7ad843ff6b0219d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/be0fb5dd19fac6b43622280dbff78c0eed318287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/f4e6dcd32552181f7f5d45200623f8fff6b06fad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/0b1bf3ca2ce1112b0e153ab3694f46da163a75f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/1a555cec534d212c20cde5b5bf4ed5febadbca5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/1c37535e76973f97834522742c0f2cf88603999f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/4139a81385fe78c6fea5266fd2cd4d33e6349392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/83da927e37757677c72629afc59c8a8124524f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/8f54e1e4b65faa294e3f1be93e8166b8e39de612.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/c400cab49405e9ac18bb43f0b5bb5df3b6a6a7df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/0ccb4da2106722578ba1ae1812ebb53e29b880c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/2b1ddbdf6883b41108fd672e17aaa82cae6c2d44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/4c211ca4dca94488ea92cf14499c5d433a04d141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/684bcd0184c0987dc969e48a7a77ec0ccc20ec53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/9fa8f945a42673af6662174b4891f5b5111878a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/f080b1b4423aafeeaad1a9386b6d79cb3bb5503b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/067de232141def996cd295e4b0b5591bf6e97b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/3700e01d3270bba938af0498f7921febc11e4c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/3a60edbd1cca503fe7ec394ed812971c65f1a547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/4ff76d01642bcb5f4aeba4d4427e71fc6bcb85a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/92ff6eae30bdaa58bb2611e454a51e75f10fc0a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/d5477b6f766d77a26ccd8f9ba6bff351faf29e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/18049bf9c7ecc477203b525c0ed054d41b780a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/2ba19927dcf3a853a9b2a8e07173dde5a9a632c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/5f82d97391964212455994f5dd65285f5349ec38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/670781408bcbc061c9b8c537e4b01de77ba8530c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/7b8f8b0a5d80d16adb3776a46fce448c0f642990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/c0790d0f58204c86f51b3f1cab79723ed03278ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/1b7f6edf04eaccc2bc9ca64ed6ae7bd2a669880c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/2e9404636e637ab30b852bbba4b6205f9ae85d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/9a0505379fc9c9b2248d32b9a6d543795a390edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/b20d2c217e373c2659b148b1e57cdb7de9f025bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/c527688a05054186e8ef422ccaec0f900c0d18b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/e2649794d63e59770426d015aa0b71f265c38353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/f05e4f9f5c59018e8bb4f7f5025844be717df1b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/03a7ccd0602039b867bdd7a1b644f889b8993487.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/0dd3087160e365930bad7ba2446175357ae99332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/2b8dc26a234991de98347533756793a1af6cd73c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/622785e10c0d7f5401cc24dd7f893c0975152e86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/6b91e76abcff972ea59ae00808e9dde46c33b50e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/7fe9f6bbb0b13dde82c38ef69246d66392b323ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/d03bd633ce8eec8a648c4c8e41c1861e3aca9ebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/464d232d04fc34a5ac0af67471c9a4bbb6a79cbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/76c340c773919bb6363d0c7efcb7ede38cf771de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/7c9d4b0af43a49e0cf08592f3b7351daf82e72aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/d1772e89e6629d6ac5c63051fdd8454f1ea7a473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/d6ffb573d67852fe2d3377ab9b1579ace474b484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/e0640ae77790569a4e9828016f1601035d8a30ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/3a0479b75eddbb364d4e2ea3e2b41a3963bed611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/6065e06b51a0a4c4833d889559b2e960871d85d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/715aaac25ca9d2099bcb8ca9cecdf48a550d5744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/a3499fc0a9699ee8ac8430d81aad79aa34f88485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/a384cbb2187cc5b0885b99d1e00803631a24573f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/da0e2866a46d40f78f98af21553bff9594aaa7dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/13eb56446830a2a539027df5690ab42c54beb10c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/460b41186fa7c569ad74c5574fcdbee30320ae53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/4c23e6e716c90b5bf51040c428d46dcf73f72b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/8ec1f5dc0b436f04231d13449c2a51bbb4dddced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/b081a2a7cf3be0451278f9beb3ed452912aa5377.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/ba1baca51f38e77b40e6306e360667b7d822cc8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/0f2d0d8ab4d744bba27de3d2d72cf3534b9647b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/183f62dde237d96b520d8f14185139b9e385eead.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/546bdbb46cab7bc3f6d5a4c99ca969dad1fb0a49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/5fa055d89f025d9a445d9bb6fa72cc7f1b856f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/86c51e869519f201dd19672a0dfade565d94bd09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/a52f68abe5e266216421916e27a1eafca8deeb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/da023b93170c55ebe5f1514d39e1f05e7a447abf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/0019d06c78c17cca1737739bfd9ae4f737b6e15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/0c9e824598e4c31a2c8af6562140276dc00f5818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/100999b6ebb84b4c34ad2b70637f17755aff3b68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/704261dfa4e08a3570ab2914b6d8c8752165c2a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/ad292238d52e5669078ad2ef987c3adc90fd0c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/d5d242dd61f148efc6294ca9cee90c88dc64b6e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/e4def408bdd7af2ea48f5121cf7d3fe548328e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/056f08e39214646601f18ae7a65669a7d7542cf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/5d6868984963e61e0f5c8f8badb0dfa6e41fb0ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/5e23a9112abb499ae6cb14aafaa14585d4708ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/6e2a941005b1f0cfb5e0b2dc9b1d128ec93bdd6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/ab372da578b3969f7e51a64a4906c3fa57b000a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/b32e60845a30009018de07c7e406642a0761a2ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/29e374dad4ddd654922144f4717b72e247c60ffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/2d8c0cca3894131bc9a31611120a216ea23aedf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/559715357efcd748fd2b5e7d0d4b110edcd06126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/89cfc2076a5a6e5fcfa5f30cbc0a5db672fb27fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/c27acc363f43ef493c42a8033015a019b4f6e096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/fa6215d53b85303c9a339dfa08abcbf3f4ac3efd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/095fef1c43a033c83f24073da7bcadd43b7e3af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/33f29b4435909946ffac236719395af1fae015ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/99572809f7556ae4a6b0a3689d7b0fa62e9f6482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/c01f2debade411e736e434e5c60c0860acc5bf13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/c7cc8017b3af74f7c565a54dc1ecf0a21ac50252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/e1766a329676f76d0091a992a90f8fea8bb4d628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/2fd0158064726ebad248504734299e0bf974ce17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/456cef9d168f953e093af19751125636367fb2ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/4fed544916c18d3db80ecb2b843fd634ca828a86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/87a3ae83a28b408b9114064125240360cb548dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/90fc61639b6fe68cdd36ecb86ea7249037cc96e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/c08734d738b6072984d004ed1da6d0c92893d88f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/e285dba4cca18cbf5164aa82d1060314920c7a5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/09f8c2af02d68b18686a0367f219c130ce88a606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/41f403d236f5e13e3f25799c4bcb535cce177b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/8ca9a8b65ffb670d74e92d3e7d50d2b345de0307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/9396772118553bf4aeb357212aa705a435e12268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/b99696c9503e7bf44ea96eec0a07025e1be6127b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/c21957cb45beebf79821efaa886724c5329cc192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/fb3522cbdfb3ca702f24d190afcfb09026fcf5fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/2e9af3f88604f905a7f0ffdd54f453002f62fa2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/33e6d0ab28f2d2a6b32200181222fb9882449ff2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/34a7f969ea6a80e7a4b631804b8732ce4a4a14c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/4b44528938bf37b200dfa727f1a1c177d3cfc1f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/a34ed6e749707d49c8f4ae7d848542e1379bba31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/f486e1f466047b955613f99e5c9e00b4b150f733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/4c99f23af98afb5a5d5c7f5cbcf6816ba38576bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/5f6f13d0f4e4fc9315db8277b888b7ee53c3ee62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/79a1a5ac523df24082a91a5102272368d5729d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/c270734a2342347a3b5a83af888d59803e3d8949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/d542aa47980e03227a767b74a1f65d023a789120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/da95c050fb9a66d15c56ec7df90c94a7d9e12316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/331100e9ade69ec635baf72a5d830b7777165db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/6aaf007a1ae22c1804c545dfb18147bb3cfd7204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/931c1a8d18bf4ea694fc7d5d60ae8aa4b585b512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/d0b7fa7e672fe3dfc82e420047a77653df75b5b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/e0dd98b8587d5d5bfd22f094ad92d3da789be46a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/f4f806ad4d65d00ef505d722adccd0075c4a0b31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/01de67b6bff5ae8f176d0db3f6b7c6784a17eee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/1002f0c3972f82721be577a2c2f0d52b08b67a20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/8f25c4bb7ecb5dcaf075db44475219eb72377bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/981f4398a75746651f73c19cd6485a5df6e06fcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/a1e312a9df1c5461d957dc2737ca4d78e996217d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/c657ffeed33f376a6447c63df3394fae9e106205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/c9408056e2545638b5ec8a357e7bb2d101f9dc0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/03abb20a1d9436ad2fca6f0383959245a7d178ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/1871cc9b1aef710547540c9f761b674c23f64d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/1e7ef92ee0f2c4dcd911b1a652bb40038fc50839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/20861c01aca485692b8996dcb377ae219b8b61af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/34152e009b4fe0ce8b357bb4fa35ed124631d49e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/8eb358e5071b9b5ac2425046f3698606e61aecee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/ebf31a399ef29765f956a59e17f537cee214d7cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/4be4f7d1bfe2ac521f320be915968ff973b7d524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/7ff17026e4354304468590279a59143c3ac60302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/827dba22b96f73e58ba18e96c12809869551cbb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/b848de268f79ece7398e6ca74b8472a1b2558567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/dc4416fe52933e2aa2a83a98ab90f68b570cf8ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/f41a18f551460c2398896ca2bd07f16d534d6c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/1810428ab8dc4f293123955af91052e10b80b45a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/20e6c44bab2112c047df815b068b4c99051c5cdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/5044d73b4d7899058825a39b150ef7b23f3ff202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/51a6910ad79bac3fee9b392c20de025945a40fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/627ce18421180302148a7a50267aba6d7ba205cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/d114d04d910aa8a18361068b77cbcdfef93a94d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/17b68d84176a7c7833c2b69df634171934ab884a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/28db1e93753f37d70b590299d6f4625459adaee5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/5eff17ca45b78cfa8c3c349b74141c22629a8ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/64326d4787c3ac1247e82ec9228fab78796b6c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/a699c4ea102601aca3b619ffd1a71198825afcda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/b55800b6e251cd1a170ea1ca6ad13d5b7b26ac27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/14c9c9171e1b6b78916e4a12b984c9af2dd88a53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/35ef8029c39035a5870a5644fbb75f180b931c7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/4acd87ff611950edd4a29a51c3084bb9642b64b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/7048c15297738c61025823530abbbd2dbf6b64f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/902c12402bd005ecaa0437c4150df07ba3c285cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/d6f6fcc5cce1ef545fe67b0dec5e36c4a61b844c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/df749b132642d2e3896c1564b4534a276989eb69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1717a748735482e2b58167d26abf12785527de87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/17356f20d1551c697d6dd8377303cf4ea3a6bfdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/18af55f86884c1db7b06a5dfb9e3384c1c468faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1b1549f25221ac2f005f948b4ec44448fefdb52d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1d0809b2e46a39073682c8cb92de0e7ab24e444e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1d50f74db1037ae6e2be58205fa4e3aa71a1709d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/201401392f602b5f45c84f5c03973e8ff58ce6e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/2d5e070d4a31e29a0b0096315887e7e039b2e153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/32ddaecd272d3105041ac86b90f820f5d2f34f52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/33e94c37be71afe10f70c8a39e152a2258f43e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/385303090f2c754f0857cd9051ae846c64f7b15b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/3b3b4f401eccdc2187d86a0647b96e648e870a3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/3b4671a7d535500f9a877e0cda5aaf0bc989219c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/432ad93a7460a8e0c8e5b1b4064f03a96e1a89a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/44b5ad165e779eee4f6552058864ad2586634fb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/456fda5258bed77854bf8314486fef102b5688a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/4bbb6904461d9830e116aa3d65d52397fb3f5e27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/560ba4ca415d8f5d08f614fbba9ac7d50c5eecdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/5c012df252dba420cff5460c970ed43037f8ad71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/5c3b5a02d16472820338db0bf82bcd789795f4a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/616662e94071b96d4d39ca6f444ba574e3eab2a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/64be0b3740a5f900adcca3d8a068c8faab8068c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/6578cef55728da948aaf4c595a759226ded2087c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/6a5a9266c41b4037703b5e51233202b98478dc83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/72f1da6c482cfbc3c6860b517b0e16e3ed2eb74e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/74b8c676db98bcee4bd483c3122ae986a9511d5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/74d7472588fb591e6db7523ade891aa724f33754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/77e298a073afffa98a5e826ea40f45425db6dd1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/78984b17e136cedb8618519e6035f0ce1fc32c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/790c349da914fc2f00c41441a8d88de4caaedca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/7bbd82c8177780a9d2851868c5630c18ce7eb94d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/7c0a71e7f1a6f9a2402ec8440b49270854576775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/82063e7148921164c5928bee18c8768e9b7780cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/820a41303d80ed145b1291d52b73ac880a28ab4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/91aaeecd722f444755524b68fba56b405173b9bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/92685cf839ab534d756d0abbbb3f5b57fa72761a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/92e7558f13e21358611960292ad2788f4f0490ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/950fe2d3e74694927a51911b6da7da1c0061b01f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/99181c1c1f1c76ecff66a2de96070d36f9234961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/a030d4a15693337f3e2a01b41b705e192a0a064c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/a3a8065fcf40b54a36b69de21b7c311afa7e640c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/a86247dc486d7a25116ce3ba7fb69525e8dc515a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/a938ed7ed6a058539af0a6c267d023c6dbee3567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/ad6291d9f3e91398d79c6ed1b079d64cbd485b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/b1a6cb8935439c38c5bfd3b1b1e108ce1b24cb33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/b3d547104da31dd7eeeea350d8d39e96801939e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c2e0b8e5c5c7406a4eebbd53b17c66cbde5b8062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c3ba12ec2d359e7c3c3d6ad58cf72f3dcd8020da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c3f2bca4846bbcbc9e83d52c042d9d560c269f92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c43f1a5cd291bbe87056e31844e433cc5d8d49b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c7132f9490e00fbfdf65dafde97e19a197514fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c8d638567b6f1102b1532c9efd32476217c953f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c9914717556ea1d31b09a1018f1cb74159ed3405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/cbafd25d9a45e971382671fededc9f38f4d50fb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/cbf8cd038374b30a16ba9a18ebfcb100c7ea6b40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/cc16cbb35df2bd4828a991ff480dbd554fff13be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/ceb78ad5b1646d12287235ac1cc63756dc30d77c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/d8f2a490a54dfaf8ff7f280c305030ea53b5fce0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/d932181be366f319b38396e9e750f176d342d4f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/d9fe122115e7898627c2d6e2632d6b9fffed8abd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/db303cc9bb096adc02a07bd23031662fae5cb06b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/e1ce279495be9d5699695fb6762b27e274de11e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/e35a080bc2155b2ee8562a08188f85ff3a71fa41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/ed19c2f950e3dc0d32242d1edf87aab2edfdc589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/ee0b6eab782bec1a797015a7be9472442472ed90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/ee324f0ad22b9e9aff86853df8eaeda821bb55de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/f13484f61e68e083d1d723141f56d1135cdd61e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/f4def36b41cc38ed61c1d8abdb745bae201db612.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/f599ed6f288e49fa0a9911769aa272a68c4bf8d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/fb6c9afe19720669d55aaed3a6c97c185a551ceb.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2021-39/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2021-39/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2021-39/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2021-43.yaml b/configs/cccc/dedupe-url/CC-MAIN-2021-43.yaml new file mode 100644 index 00000000..b54d43e4 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2021-43.yaml @@ -0,0 +1,741 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/06cb3e3de15464866db449e4e8a5f8ebbc640463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/0870fa4e3ac433c83df7537b4342b7191527f741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/092422251433b03d8375327d96fc6da70f03f8d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/096ca2ca97bd61076c814c8d2800f147164ba59d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1151a7751d7a8ceb3c0a09638f27ea050154417f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/142b33fcb99ee8efb19501cea5b5c32459934e9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/0181f3bb97eeed5042f23b1bae019a8d1b342364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/30557990bd22464b006f33f76d4a8107e36458af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/478b5385e761aba4ec923938435e68a4dcdbbee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/6ebabd3f265b8cbf9b41456e428ea13654cecfaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/72a31a286ce3320877e2a56ddbecfe20aec6baae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/c1d55a33d416b14bc3722df942e2a2634a5190e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/fd99c426c24e430287971a31ee0827eca94da91f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/06d47603bcde94d4d34b289d1d87d38f9cdeee77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/70b94692806eda9faa156c54aa1da4a8f2f47c51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/8623173361938660f5f8de3e122a01e30c4b1981.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/d5892a35f852ddb6b9ec5a1e7cf47a14df3d00fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/e6a8047cedce51e8ec013f6b8f94a0549f9346a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/ff94e18a619f2afdb6235a680ca943a6fd65a79f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/14fc4c5e3fdca7164d206c0cc1a0112d47e09b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/1f373ff81d95b16f3f2c8823f7f41374fe819528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/2245e1662ff8206e6371a132e93fc8d3b7543f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/57c736b595b02816b600466e69a9290ceb1d0479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/65eac39ebc0989ab995a6469bf274528d29bfa16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/b1aa08f957d44ab62386da1f1b0523b2c1900c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/379c9898e5410aaec4226a5b9691f7286e6b07ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/55fa87649a6c0d46bcd9d671102facb1231d954d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/855f49b2ee0fe4575848d28e8ac1e043811bb683.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/93fe5de0be60d80e54eb3df969c6375a524effb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/b75dc57f061078981121c075fb7e1a8709a1e84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/fb2c994c000647bb7f59b070b93b9ca78900d437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/08408ed68cdc411e80a1af61c2c2d353bfd46212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/31c8b99fd9dd4b60cd0791fc428baa5e4e7fac5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/4a100acb0cc709c99efb3cb69e161ca93632ae65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/593eee75e69fe8b411c27383aea8969cee978f79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/e185aabcb991ebca714a1807ae3d8bef6bc0e14c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/f0448db62b4e2ce9e4846e771672d27c50865c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/f8cd7b7cd33dc624af9a5e68c61c8f4a5e2a20af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/4262adb448ea180f06a539af19db0efe86f0aef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/4d5be358387fecf160cade8c5e1f8c24db893897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/62692d7c34e74caa4987f875e26bfb8e8e2cf253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/a2a4cb172f18b7c1f944d3464383a31b37ef8dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/d448a2e0d79dfc2a23fbdd86d42a0f030850e520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/fad96c57679261267bf8b020237a0c3683e69bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/ffad16f0304d89f07363cb7d2eb4faa382788ebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/117e44ffd853e7da547b99a0b571a1edabe5ecbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/18e0a08d3f596181afe3298bbf9b90afeeb508c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/25ce363c36f818b29cb71e79bcd35a8a64cb9bd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/3a5d349c76e5646aaeca1bfff7bce1af6407eca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/b3118a1e42e5e4e639046ec36659b958d8410ae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/de57a881530dec3692e1b327eaaac4d337b27f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/3d6e2b413cfbe03dbfc25a76b7454da85ab5aa8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/54cde84409fa134cce18789cc4d12a791277e7d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/9ee633e534d06f73d6cf0cf60df2bad96f6f7078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/a413b944731d3577cf45ce2797102b602218a610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/bfcbf9ffa3b2a9b36784191683d77ec449798be1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/c860cb572d9e3e3720d6d5424b9fdf1f6537bc0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/03d45349635c1501071b57f83b533b6c02cb13b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/6eb4b37336660929bb790090ce3d72635645bede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/7f4faf0072e7db35f897f8d25a73d1b78b88c087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/c065420e657cf5c23323ce1b4199fb78f544f89b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/c66c02bbc71803318ad3010b6903b2ce3ca90f44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/c91da427a0f212526938a4081dbba28c2119128e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/0ef317a36a43f148084bbfacd944609be94dd959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/19adab7e1aebd10554ed1d62c1b6f37debec598b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/71b0ce66ba70d3798b5a66ad026563e8679a3ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/758d47cbc44b1a56e0916dece8fda927e7993416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/7977883734290f09b9bc4efec93cb607e1c337ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/971383b7741ec415316bd022ee8acebf20aecb19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/9eec5be4371a9c79ead6b233ec2e8e4986477004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/31d3c25407a4c2ef48a4585993c14504c0cc9b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/367bb17cab83b1acd13c24cfb7d3731118abd34b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/4d2b75d2566c941c7808f418721108c71ac4e426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/7ba5e129042465707ceb60f2314711218457d27b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/d58d144ba1706a96269aa17d657eb8194ddb663c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/e0a0cad87398254dd37aebf8506d83fa4ef5c5c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/f7af0abe1370f7c480b8c7b4b9a1eb47757630bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/005d8c4c641b362a9d526ec308e15d3faae8d983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/4b661742b52ea30b692057a23119584541957c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/55f4b3159f596ad0291e9f7a4ba4e264bb60e264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/ab0884bf23617e0198f9b0c3aa6edbd6145cb539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/d8a45485b8ecb4877b1b4672363129150735e20a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/ee40e7d90fe4a095ad1351314183e5d1d83796c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/1d04eb159b9478d6ae42bbc0a07d64cd3175182d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/21857c684694a63a5507ae20f0593f03bee5ae29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/57fc7aeeff7fc0eea77cfcae1e3abb25409664a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/b6e6a335536639447780b3fcf037e8959f45107e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/c3689aab479f225028e5421d086c55665b814399.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/d99258baa9eee50da81ee18265dc5bbb26d2737d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/26e0914ef54fc670888ea5014c9a940b3f3bda80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/51d39cd310223cec19726f2ab474fe16d96c0a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/681b36d30c05f315d269d33116c31b863c2da30b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/7d140651f45db6072b36a45701509f19a01f5b92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/a24ab4f3c3629dc0a3306e8f55fbd0ccae63ff11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/f3782dbb276d3689dd48a130462b863fd96b0dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/15556bb38ef198b3e764ae9e690565c5b52cb166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/4967c5596bc618452fefe94e6fa87e80f0a2c872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/87f292fe979eab23e97f816a657e48341196dfb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/c0b11433f71c6b5105209cb71a2e1c8729060f83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/f352584687bf00a7d6cff042c08e54f5ef7a2d0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/fc61bdc0d6c821bb253a555ea03165272452d041.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/ff64b6aa4c4a063a24373ba90db70651d19b28db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/1341ccaa0e02e80bb286ddc9d326549b80742bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/2237969ef0aa864211fe50d96e3e4766201dc133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/6afa7a04ca84be32287f4dc8fa7fbad2ca1ba518.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/7167ad48ed481f045b1c4ee2cb0cb67e6a05cfc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/89e21a46fd45fccbc155037b762b8d0dffc87e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/99e2cf95a1555632e7e74fa86fa96776655a35a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/fa30518ef19c6058b841791eaacb340be679a0ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/1c5371ae5ac2e6bf78da2e00e7a2a5b9f4f6689e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/231ed2f8fed29e9d828555c335e7416386c3b8fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/5d2d6301db6a32eba698eecdab662ab5b2caf31b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/5dcd0a46c9af07321fb531335c53ce259dee4709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/c3c0241583920ad9440f7f3c5ae1579eed8e1036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/ca413d794ccf02230f4db78c3899dc8dc7dd4c32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/26fe8e02bb82fda6ca0477b2ef2aebcfd5c8255e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/413cdc7ea7784868bc66489746eac97379cbe681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/5a18567ad067af024b54ddd806f5a807a4ee4642.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/b07edca4fe0937a3de6834300ae946fa1e18670e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/cc4efa4642ec8aa726e6b95a27d86a5aa2d429e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/cff01613198f5a99d5b46203cf7164c99f0ea1c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/0606c0c94cd1f8110dfcfc3fbf7445b8824e1cd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/06522ae793d4302823c307bf5441b4aa4d5e509b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/3f6a66ec6f6f4d02873aaf43c8b812a245579f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/4501c06d30f10809d2b37be17bbb1d99335f26b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/45267c8f76af8dd871886446d2dcde45ca6f9530.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/4da032e8d8ead00edfbfb2ffd49e6468852d9660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/1065d7081134660c9442ff897528312d415faf04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/1fc6f89555d4e98ceb3f46a530173baa61cea3b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/2149f5d8519de9ec080200526d4b5794da54389d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/4c6569fbbecc168c55c82f2a37f7603fe50963fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/7ab5029bf0a8389bb0abbd0a9c65a850f8d58d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/97bc01d762cfe7907c46c477050051ec909a5848.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/f2005a334d931cc40a8f31369fd8d4ff1a01039c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/0d133a5f6817218448f4c308924caed6db57f98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/0fa46e615d5064d936531591c86e6e9d1d9daabd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/5e7ed16890aa016bc4bb3d794fcb4601fb589de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/7f4940edf398ff4649b693f7f889da62e5e94e25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/b51d31ca4ca6a652ef48fcffe32646251f2ae371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/cd23a00526267e6a69bcaa7bb61a0aaed7fcc990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/dace5b4664a6b269ed154331c8aa454d4e624df5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/44d4db959c3954da034dbdaa01677349c0ab1cf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/709115c40da2afc3c6f3fb897ad01984a94bf979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/9459613073cf73892e7565c8161910a8680bd6bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/9a4b9159de11896fef485fc24250f8aa7be9aec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/ab77703b5ae3c637689f9c8271fd869ea890357d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/d70cc3dc16daa122378c47397b5afd7398e5d3bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/47e4215fcab74fe603ec96f7881ec8ba2b296380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/4fd52660bc58496d7ba862aa37a9a533f1f8cb2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/973978848dcf7f77ce2a0ad82aad5b3f2e274e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/a683dd24d655a1608511b84fe211488bbbf9a4a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/b20985969cd92f10e8ffdb0121f0ae9851040bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/be7b375f5f1cbcb495368b2de05b0135b5b83cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/11af36ef2f6c5ec2736a3f6189b0917809156ce7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/12cbfb7178970f6af3d8cdc8c34378d36660523e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/4138ddda31bf60538e9f525a7602c733dfecf815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/742b99c3714cde032313fe7ec31ae7045721d029.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/96e997ee5c7818efaf8dad624eeb7927154a9f59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/c793585fc063c2bfa76a71cd7886ae3c45e79c68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/1d0bb4f2ddba4e76b0b4071f96c47052299eba9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/25ddbeda35a7f57d8136975e1d0f6e284a2fb323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/4dd40942a73f26691dbe5c001ac30fbe5196c61b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/730c167f93fbe08757dd616f19d5f508ffd3c38e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/9d2971c61af98a1fd5006836050014d2a6bf5e52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/bb8280cb8a231517a09cd5384daaf3c29834721a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/f2502cfc3bbee5e837ebf49e003e1f530b5d1f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/2c566fafe0eaab169d8d3f8fc9881f05e93d13d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/405b10af61d4d6a01f667281808f97380e9d92d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/4d5abc9136a48948e46b4ec76ec7ab8e9e11bfa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/7d64d18a4cfd9686594e707c037d935d3c2ab616.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/b1d11aa31e0fd78f974fae0ebae1a1c26e2200e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/bc9ac97e0ba2fd19a407e4a2dfc3067598a337ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/e9621b9eb19d1a00aa17b44ff8cead3e4950c146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/00c7890ebdb183f7dd42fe8d3d4a1712fc043fc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/37c3ed8a8c9d5b3b19824faf3ca37adee70b0089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/5fc05278ea242c99a6ca5cefa8b74c62b4b94351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/b6bcc433e1f285288d3471a4aafba59937f7fb92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/bdffe6cb07f2c955fe63beec7193e9f32427e366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/d3ca38dd8690a2d38f9812258f2806ecc2b82919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/0bc12e3f232e48364611d8ab5e416267baa09699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/3d48942e3f5f90cd5e9b2030b30249f9c8597670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/455bb39312efbb42c4f118f4cd10eb5fb764f75c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/6aedc7eadb36572f5f48f6983f1ade8ef88eea41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/8b5f9bd7a6a3e14c51d81434bae4fbd6977f97f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/c0b20465a82f73141e2024cb64adcc936b46def1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/9b2184ee494588afdc124609acc0fa7731719762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/ae5162691fac8fcd438cc47a5e14842f573b0374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/d0e7fcf6d25509591795ff8c1c855dc6f9c42ba1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/da1992493330e1af6de1ae1afc081d124779cb49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/e6622ea75d666cc11a0b91b87f9cab2d7a5db0fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/f620ccd39d35a7ea805186acf730baf2fd7e20b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/067f504188cf0bb4907e997e1b09585ae4e4858d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/188e76e2f74bdf69c40f9919a84d78fa8ecb81e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/1cb48c6cb6720363fadd2d5df762b6b59f63737f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/592c8806653d2ea7462b04ed8b498dc64020b4de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/5b3346c5b58f4d3fd66b385d21bbd8868e32c980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/611357c42f87eaa7e91588524cc999f1c56324ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/8bc2d090dcf49fe811b8626c6148b37987646b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/168d1c7adbfc7d8f1b0ba3d2d6de6534392de6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/76d19da4596f85a23319ed9baf78f26c3f991768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/97756e8121e392b93e7d7e929ddb48105eaf4768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/a609c9ec09f869ab630cb18388db9e01e1a26952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/ca43cac328799608e4a3ee32a7d72337badbc1c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/dffde2b04e8ecf0fa79a6e2ac2d74b1c30e3183f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/f52cd524211c3eb5a319bf941c1bfc144d04b25a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/3cdc22cf59f038245b1d19c2a980679466ca1ee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/7589d69380cf869762fe41c7722584bd249e3447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/846d7bf13a2f82518ce1108f7b0327fc56edfea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/a606370a6ae460b2d5b6df0ee1b6c6382f7b2647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/b8fa6d77d12ff483e45890ca58382154ca852023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/fe170003433622ac346eecaab468ae6a036d0438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/05167d22c5191466a29818d28b8b55164d1fdf08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/28767e272b3196127d3e05cbc50c7425bbc3d73e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/2adc9f79f01384b15bae0c9d6088cfd48718096b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/526886fbdbf7222f3b005f16c3e79fc4e3c84092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/60572bcd9774831ee3ccd4fa3a72f20da16c8102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/d4e087b3a82606e4dd2509aac80f87c35e9f6407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/1c9764fab2ef070eae21fa9744010a21bed1d79d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/2b652b821d4a9ab6976b573104e40e330fa03625.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/4a4eb240a85ede9d0bcdbd99b38b0509dba410c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/503919eb59ec1dbb2f43d31e3a5bf0bd02df8f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/50aaaac0c7e7fe66f6a4979b18ba873ce98d474c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/8980970774a94c9c6e9e92ac03042e8c8af2b8df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/07718b06e9b5591a4c875eaf4290f6971aa80c1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/1ed02d3bb5eeb8512afb368ae648f84147075dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/3557c8261ca3993b4c00e50a352d9b1bdcc16da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/6ba086dae9c588ed76687f3e4bad2dddf170abce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/b2dc26927a7060e9c852f69da380cfa0a8525dc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/c889a163170fd8091093583493fbab7072146f7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/d6c89a1633341a81c58062c03a217b6c39cca78a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/04f1413ce85b798a4fcf21a7d96ed7e3faaf6747.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/37036d69176a2dce54a0d3d940ac4aceffbc309f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/68aaf56d296c790939779fc24f081e97d2b23fc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/7a49c00ecae29ca58f50062950218d3b8778f9c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/8ac771cbfdb4052b18a4fc8b62ba3329fafb073d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/e52b1f541d500eb55f92ba7083de32ffe63f0ce1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/f50aca8b4aca409d86f734c7f092fd0a172b42ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/063aa4df521f8eefde99ca5edac9c89bd2348cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/2013b9ab053377fe9f5a5b246c4c92592b57aac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/ae1afee7fa86dd6ab5fe9f858621e586c1c59645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/c115acb113d3cf15a2acce33db9bed64ca401537.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/c6aa790ff8a344d8f3a6ef96ed546cdcc9039cb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/d36bddf1959812f47729903f7457fb8f8784fcf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/33a1626fb98325d954badd743956100e222a8a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/858dfe74a785f88663f4815c7e9d47e13854c980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/c2b42b5035ee00e3a1418822106ec33efa2a66cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/c9bc763e6285a57f8e2e8266b4af6acaa67e9faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/de26bfef7627abf13b6a358f656113e0e4c7cb61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/ee97b27c537db83ca80e315faba951e9053fc1d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/32f4e59a5274f64dfa5450687962fc04a86f4f7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/6f8007f6456d17f1db2c8981fbe180fe52d923be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/b0a8ec7ce694358d55fe2f39598552791c4bb088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/c5e2cbe39bdcda36ce884548d729e400e55a47fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/de12b851c87740f5f69419ac633e92d42f4f3dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/e57d39cb1ac3efef4a2abaf8e09c36f80950057c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/15f23925e0d1eafdd584b3b795b80cc7becf0b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/1b046ea5a8a13ceab762032cbca6bb21c8265365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/3b4369a92f60aab2054f502a289073bee71c34cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/779b041d6d4a07473f65dc66d02bf83e26ff4812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/d15cef5947c09ae4475fc330b3b053ad3cd8dc0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/e010527a539adb55cf04a0296e6977113dbaaded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/f910ff9da71e2ec7701335533988f8d4ab0a28aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/337226c753ba8d370397f490d5dc4ca35396bb80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/5a598ecbfe0fbb6c4169e8d7a0ac53ca2d440638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/6fa25621b0eadd607926d9c9a18f49f9e4052608.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/7bb21140d6e49edfe220530e366a1f4d8af4b20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/a39c8c739f45d5b67ae4a3a8f7cfcd23aaf32b94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/ac87c5ca609cd01628b1c5f2f3541a8fb487def7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/d33367427ed6533cf7729f948db0a7e8c7aef118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/180faceeaf5b7cadc2832c2e24341d76deec6082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/1af3932900b35b222fe2028b24e6d54eb611755d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/201160e32b4df54a415c120a05bd5e21635e65d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/2234bb1c318c314e9df9ed9aae118778e24e156a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/86a577446197a9e2ba66a37b4b995cbc5d0845fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/e224b32a6ddf01fda893405f5d1545a504da39b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/218afef50f516de65768b09144af91728ad3c27f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/4a01123b50b47cc1b587796bc2ec03100c34f0c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/4c09df331c3c277d4fc69fa0801f0ec2173e6bb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/5c4198eacc22c1694e198e0f6c7b765a823b96e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/a3b04d951ab24d385cab86ec60058b1926958aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/d0afc436774b40d393252fe1b95f72761110bae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/12427090fdba331915377a31e2f7c7ec4c212306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/24105b6049259244af6232a5f134867a0da8860e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/84ed0e240755be056583cd00a4b1f9ac5d74b57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/a1e4bc0ea9bcc9ca3547dacaaea2ef5ab58cd72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/b3388f20971d547824e47765c44a6ca5062a173a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/bcdba2134aaf4bde20de7064ea948b9bef1016fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/3a55a8dd90e43ed0183aad4a34acdcf4e0c928a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/3e01350eeff18e3cb3117f88333aa6923b6b035c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/77f7ec5ee9d6e1d6b535c1b48709c191b3591707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/7b1a385fb23d3c0605a039f9572955a67f4b1113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/87b66f25a848cc75f72f569e418ba4a2af4d27f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/90e0890411852184eb13f270295f75dbf36490b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/a2225fcecffcf54db949e0e5a870b05519f84dbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/2ad21d1dd942cb5ffc5daa16488473e4462933a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/502dc06c3d60426f971a0914c8cb500ed96461c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/8dd590dad630986ca100390a7a0293b17696e91a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/aa92ce9fbc51b824da8051758c411179e2e14d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/b5ad07b78c5f317035b72456061ef29ae608e163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/da981cd227bd19043d92341851c2b30bc0c7d09a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/e4ea5a42a7953eaae729ac30be9fae25be30d036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/0c9203d6d6bfea58c52ed0727125282b3fbaf836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/0cf1bf1277449392066d4148d5e4e4fb2a37dc5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/2e16b8f39e95043072ce8e52c04726f754b5e204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/90cebd3686cbd653de6ed00513ef3bbee45dae22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/b5fe91f90b39d576fda8e6dec30848110630a023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/f3af82246f90319bf15d3b72bf5fd6e819f960e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/2c024c39bbb4857954fcaf6ca64bdbc48d079100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/3f431e1fbbfe64ba5a5887e4ca7de630e56d4065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/4e4b8e6d2915bbc6a285174df8adfff89cca28ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/5ff6cbc843c9841e9d4bd0c9a47f9fcddee011b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/63e8fea66cd18e937c2db684702d983e9a893684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/7f0fd497050e221d219ba43ef9ea9e95da4a9cb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/2a47fa50568fbd60a4a52bf719e21b93a70598f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/2f61166aef294dc97e8dba5ee4f3946574a5626e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/66292d4f3d6ccf2ceab7866ff6d7e184eb5d04b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/67f57b0a205546a8e1c70b407e82990cfbc6e784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/84f38c10eefb3e8d6acb99ebabe1f78aa4a731e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/f50b43298c6d8c1446f0bb2b5421bace2ad244db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/129d892883c2fbd225eb1be5f3f50847108a60fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/38bf51101160b92c42b4b663b1fbc131a2062af9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/3bf1798a6fed45175d156907fced5ad5ce15e24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/3c1757d9238de29de21d8ddd1ea5cc2dfaa41ae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/530e07e79758ec3da5e8bd6a6abf1e699da4bf1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/761b9dd928b1df1f6aa883093057a8ffd86676f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/769f89c5a08be28fbb583d1b1bece6fd5747df79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/1865d9c838531d06715ace2ed7190298324e407c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/2a1d8bcb881d77a12e26787b15edb430b4970a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/2e3697bdd53b5e72c6159bc659e5eb1cd922a5b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/5543aba806b4a4396055def858c02291b0b80b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/81cdcb3e4b67c2fcad3df1ae0455c9bcf196ec8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/84de1d59d6730bbb580471693fc24d904972f706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/907aa4daa7a3ad6656adebaaa988dac99e1c8dce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/3f19e0d0d0cc0264598f6750be9339aaea9ee529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/8446a39ae84a6c85e365d8eea78a14daac49e5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/acd2996365f546a0d95752aa65c346c04b2d4302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/ca802c21a76d01dd868e73cda79c440fa1cd4f58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/ea4e2ba15d336f146d90f37e876224b5dc27aa70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/fd9fd7855d8f1c2b192e90a75829376f8efa34d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/2391cbec1ebadc4a762433e5092a9e24e8c49db0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/5d2a74fa135cda1d6670077fca1fe65739d21186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/68eb2637fdb465ae1a7db336698611633ce60c2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/79d2709c2d4e1d59b106476fc3b2b8b45b19b2bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/8e9ae89bb437a232e1899178d166e723b7bc438c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/dc81df472df254316364b957dbf1af5e5f3abcfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/04ee028702abdfe1b80655edf0215e0422838d8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/1fe88620cff66ec443145556742a59e865098737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/5c783c5a7ce9487b6427637100e713cf41788b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/67907ea801894b602bf23c3d7b1c5feb127c9ee6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/84c6d4d3bf6103dbc35998013a4ca2394421b534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/d86fd7c5036386af10e2041675230280394f9a34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/108f02e87b1a1734c88d13288bd4609792aa1c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/387c23d36bda7217fccf759f34a421d018440547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/43a8c7268ad3db8834aab7dc9617417aaf08ca7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/6e159f2da439e83790f0ba9d6b8bc24266fe705b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/7f3360470a31f96e713cb4503d5b3a67306c8166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/b52737d156a53b46499a13a6af6030b181cdd3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/dcb382d360da9ad3bb106d315ac3d3a234e810bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/1f38dfbc88438adfdfde78dbe4a460bdf18c6336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/59cd2b3a7a95a58ad5c3d874f43e67a0b0470973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/6a3203163a37c16474827e5141ed4af852e963bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/70f131d58d5408ca2999214320f96d5b9bb453b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/88777eb66faea3eead1185792a4365fea1a5a728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/caf9866468b841501821d824d60e9da59c446e1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/d6f1bd2d215111396690a0e58d2f97e10bae49f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/02db1590bf8da2f9ace948a820f9000f1af02384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/178960ae09949515d6be50c61e28a58328749722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/2be439fa94ac0bbe84a0234a10de87d8cafbf1d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/53ee88c1c913555f63c1778d1c32c1de3dd35080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/6bbf6d880448b3cf2ff4d79650cb710b436044cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/f71728557426ab34f46a9b72705985cbeeb7dc14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/4fad932e4e6bbbea800cb04196dab94ff83ba37c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/79cb46aef34cafbffc1640c084532abd01acc451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/843a26b1492e1bfe413bef4f0d9aa869dd501bd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/8848af6309f84d19f9590eab7f86d6547202d326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/92e0bb798950c5bb25726ef53fb07a5ecafd910a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/acd373a1d79f0690cf9cf2206c03234bb7017f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/08f163d0530423617548fcc487c3cf538b8b7fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/760a6129ad58a2c74d7ac992f73213b1b872fdd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/7b558018c9bcc185a89842cddd446d724b06ef78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/a2c619f1b25eba6a771508fa4a4820353566b278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/b2440863b01f540d50b5e8d36067020f344eb2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/e924fdb11584673c12eb49d9df66f6277a62f2e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/411d3af4ed50c079d64e1ebf50e8003dc5454dfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/4b8f1d2e4f7adc936af0f32d6904746f3738b589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/8f1e525d088d824a277aa0a618120b25b1931d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/91a851017584d6a2fb2868020cedc6598537f168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/aab8907ca598250f90daeff51149f9eef7b081be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/bb6089d67a8504d0cd99674d13bf057cc6a8448f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/d2e0130eb2ab28a07582bbd765b011febbf88620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/1b4756d79b0f7e05682fa7a2dbb4e0b56d4f0126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/2addbf1d769e35ad83e5797f58cf0f1862d6f2fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/41084e12cdc4163e56f64d672bd14b11ca9102f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/67de7a5260dbacb81052ad61e42bec9ccd014129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/804de3e1478ea537b1c6ec6390b07125e69cbf6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/c42bfdb95df087eaf367b72060b8a41ee92446de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/d52507777cd71252d334b51a494f51546f3e87a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/25e604b73aa643c4c95aebd4a54bce085acfa16d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/442ef6af6fdb9a550f3cb95195e97a74148a1494.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/537d8b1d12ee8345e0b4e55d4945d19a0c443316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/b45fd915b1f6901b800d8dbb405d7c7de8c163d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/e9b1f0f4d815483b9ee180cf75ddf34079cb6d6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/ec47dff78c118420bb908a8ec8c18147aa4089bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/65fc7fc95d9d6122c67de33fce8a212ed21d2b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/b0410e4d2c10e033e74079b0bcae8e94a51c7891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/bddf0a5c7d1e74ca7e267d2219cbc7b583d37434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/c38a28b3282f8389ea228ce5b2003baedd39cd7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/c4dc130124821d2a86827b193563e08cd0725d8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/d49a0038166ea8d5f67dcf24cbd18d1f34581c0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/12ffaaf1bd7f3a269097880789aa4f394d1ea6bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/13c15c39d2b670d33301053e71e32e7d153edcc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/44bd886b1b4331a152e7483bff8ec782894b5714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/4d5fdf507106d540e387c29bdbb45bc5071659e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/4eb2bdeb64be71b4acfd97bc865ce440246b39e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/a249cf4f6b5279f56586c81b78ea5d2c96391963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/3a030fe0e8b814dc83754bea036d58134fe88025.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/73efce11404a286fad861986a77476782c3e4aa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/81ea29e414fa93d2cce69baec8b1191ed025c23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/97b73089aa491d6ece443747ad8e210b3339dea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/a1a02e2457de25026fed1181c1e76bb235f43af6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/b6dee61e9abed36a10bcca71c57ad964c6497a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/f903b229dd961d0e9780b304877fd4b4a4c1b2ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/3dbfa90cb20642d34ea237dd75d358c987fe5fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/4e6914750ec2354e47f55aee5bb248576f12ecf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/63d78204e8654196e7830f6b0a4572ee443b47ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/809bea68f41a7857ecea197ea334c47d7d5fbf77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/a1abe23fe36d96f2fd06b348d8c1a2c1f9c6372f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/da41453db77a3bb7b91373125e78a149af2789ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/f41e4fdea6d4237b7f3929cf22c3363727ace4fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/046fe226d353e8b93ddd5945679c01fcffb5868d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/1e22f987c16f1f89538bc420a03da9da7030db77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/72fe21722f78962984fe26a0efdc54fbc341228e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/9f0eeac50f545e75ff5a42cfdc586e10ee5f4010.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/dc39467beb74b4a6f6811d9b61d56be1369279da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/df8b42737beec4b1922a9520f0800e38965da1e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/1e98ad3a56a32b41df17de0aced8e55ae05e0e67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/ab4a48d49e6f4728698983288c2b6dc572d9c6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/cb35d9a28090ce7b85c3147c8dc7623f0d7d5579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/e95eabe5ce72e790275982b079b46fdba2490dce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/ef246eed12bf9fc588da69f980a62a340246626f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/efb581aa303930dbd7ff0463d868dd3dbf4903d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/4456d1a8742cfb098e25c8dc7d2a52b52b776716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/6835d4b98b8c19cb1e82ebd58cab709beb25b303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/8d3d13c100b12117ea2cedd047728632c71681a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/8da5e3e3bb58c128d6b0f618d3c3400e77fbd81c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/ac1b4b78a7ca29b48376610298f01c5305bc2a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/c45432b124d7472bc373fe688915e3838cbf3e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/145231101c8dd23248bc676dd6dd4c44ccc16bea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/159795c06445f4c0a28e0252498554057814128f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/24f821804b8df9336e304e9a4b8e8146fc78dab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/5e8eb5db58c71d221d7b636dba92ab3872af70da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/72830c2b18d8ed364a00afbaaffb700e10ef8c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/be214cae489c57851ad917938efbf6cda8fd72b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/e84c13dc88416d384f407b537959a63b122c5c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/28d65dd2675dc16cf2b55fc56510de8c1b8fc171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/295d065818a94343533443204fffe7852584f310.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/3ed6fb0650b78a842143a832cba61a3239ad2d09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/56451d498e7a13ddd603b163fbe404f8f3c80d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/6bff2838741bba03759e9d657e8b6ec4da5197bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/7bf9afe46f32f58a5259a81cf4f221ce4b6d0057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/96b05cf37cdea6e4836b027e8ccca4dfe42b1349.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/103ce0e712081bc5ea7d9f31181f6cbb76f64498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/3f3f4e60773612732fb7e9ec8bc2400baccc6330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/aaf92b5c6e14a59f1749808a0d4a819dff5f31f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/b0b2b4c22e362b32e31eac6444f71800f3f60515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/c8e809ea1c532f8dacd70610d632214cc0f3e62b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/db1c20367bdb2580b007488b246df1db6802fb04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/21daf06ca22067590d9a807e9d07c08be8456855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/269b2bbd7c5d5cbb68e7d9f61727b84375439b2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/38208107e10f1605a6891adb049aaf42a7191d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/48523b88ee0a7816167e663ec0a3a09839706c7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/ca81fb4fabab8694f895e689210152d589be43de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/dd0b54bfe87d5bf4d35bb0adf64dda7012d47181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/32053a74031e227ecd00dfd7ffb42e9c173cf964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/425f943b9cae64289399f85d6a84d69783b8031a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/577bc6cff40d008a05cd27e348836b0245ae7548.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/96a74916539f49f24949fa3ee3101bf65fa3a92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/a6d910bb9ad5eabd663bce2160066066465fc74b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/b4fcc5c5039c586f5528a681c07adef967b073ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/09166b529aa7b49f9a28266ec2a8ea6e764695e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/1217e846eb2312569101f47a6422d09662f862a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/1c265093f8298aa741242e2add135c613b187c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/24d4354e96585187d9bcfbfc195353c0324b24d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/8fa3ab827c0ba508e0459ebe89efbfd13bc4a7ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/a086ceafef21defcd4f32930646ef995accf7745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/a3856ba1084dd7b24a9cf98656840208ce484459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/1608f1a96a62c254bc64bbdf972c6f2da6147477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/19d7710decec8c63687ad219e3fd6f5bcf33437a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/63e3ff888b66389121eb3113dbdc4831dff2d210.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/7c5f00d0440e9d67f495970a5db3f12fd88710c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/921d7c972d5040d7c11be63c01f2d9e4cddfc493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/aa9699cfac0e260c9362f53b954b9b130ba9d047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/d2d9f37c1216f8683a90e49d5dff9b7bd3d12a2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/467d3db4f9095117a680db27ae322ba90b2d8d0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/6b93f2825e620dfeee126af4061b08b9dee92572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/ac402714c2de90339feec33710394fd29af1be01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/d6d76feeaf0d0587bd7060a0bb17cca132b37879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/f1fd41809e41be756accb1632b7ba9e22a76739a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/f8c70aed9b2b4055af0655e534145e1a34b93050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/1d57e14c4d2d0673c8902c1c6117d6b60a50cdab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/527c25b1ee1713535ecb57418a497c04b9973c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/64ac016d20202a35ab5d9e0929e44d1ea55a1c60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/a33fc26a68ae0007c1f9aebedb602c97c41c634c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/a60d827af34ab94be7a95b67fadb1d73ce9954eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/d71a5317a12c8a5f8660ab2b98a35c9137b3d01b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/1e71e61583c85708d0feb1718e09511c80cd6aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/3ecb1ffddbb3efbbd0b9b5fe62f1a3336ad6d8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/94700003dbf7c72453f7901cea134157a7f50538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/a0eb57cb813d2ac522d0418ba0c5c4af66ff5d6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/b8a0903efea740b0f54ac92f9640fafb88f34c65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/bcbdb403cc0d3401918ba9d11e8a395da53bf27d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/05526027cf63d3a48e40f9065a743e7360dda2c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/07bf0d84848d2aced056736375f145c4184e0fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/4a471d313040ebdcb3443cb1d9e31f84dd239ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/4b31a978ea17843fe108202c6e0975a06ea2b062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/4b353048a2c1be7bfa236c7edf68aa2590282d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/91743da86f865201a040031d8ac57e706a528e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/b556f3e698e7e24e1871c0be4c4be525807b7206.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/735837ff455f7b6aff8902c5a2822be58a6655fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/7e7d43a3aa208f9b9c12062a64c22be944442a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/a5fd6d75c8e99223477cda82853ff05b2f627644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/a6b868bda40a89238ae5d9b735abe1d1a770a17d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/d5cfc20c8a28cea5f5996c1e3c72c2081094fa71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/eb27c28c75bf50bc39ad63242edf83864e47d464.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/efe753a67b0a38c7cf4adf1cd44482dc9190c0ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/0bcc88340e282da7565e165e19c885571361c7f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/4c7f9f647b5bb03b68f53695ccde43fe34f3dc8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/5858f25c03f4af8164cda93e8011fa16e591a775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/873d15c5486678660127d4ffc2caddff7aa0dee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/cb552ddfb339692ee7f5db298c06d554f08a9224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/e9976282cd23bab192598fe1efaf9952514e4710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/73ab29ba76b1dc002478c3a1bcaa4deb973f87e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/7fabedc76fd42cba64ee6b7738d5bdaa313bdbbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/8aeec029f708de08480a9858e32b8ed6e31d4cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/a219b8aeaf51f45759d4b6fb54f69155f4d83f34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/e2d6635da39e4ad2c6bd187e23413dc7302b0a67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/ed778082c97b430c450493e00f7092540c4cad19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/7483895b901b22dbb63f46bd9328dd05b76bb97c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/7b8fca72ba4c0883fb3953394faae108eaf2bf41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/94f098444e24ee5f27590d7ac64f6aec5f03dcab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/d1e1a9d1252712364f4a89a32535c24d0b972e0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/e0a9589703f0e575a04601f1fee360b8ca0c6137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/ff4fa6ed2d964a1bc0081f34ab1075d31bc4aba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/1b1e2785a212e7af5fbd56fb22ef995b57cba16d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/2f70cd5aaf813737104cec0207129e89ec890c47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/368469a31825aa210c853540420e38b12bc994c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/5cfda94974ffd0250346d2d5a54cb14f02aefe25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/a1e681b931927b365aac6e913877c6e2156591dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/ae2c64d9cbf0758f07fa89b3152e1f10f26e79fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/c5b0532a22f1b2d1e05f526c59177e6fb748e41e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/1adfd8fd077c8ecac7f67d8757bca27772c3b22d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/594fe5d08749d24920795f7714d6c4011ed9b9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/749a158b8f3493015fa2e5d538479976a747a8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/900968b630247d2d29b1fb65c1d780f1b9ed4b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/bf1a67866513c644d9b9cd7c385b52de6cedd4e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/edc94e6ee03658e2a6f19c6516eb4c046bc4354c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/f926acbcabff38b1c166909325b9fbac0c8dbbd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/2c6c355ad570e541a6812105c4cf5d2cf13f020b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/71591ae7b014eed4a6e5c4cee47a7c43c9b41c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/7244b25cc02a9449abc5f107c3a58ee29a2524ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/d4165a390d596641d49dd0d4c6c03a1acac62391.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/e34bfb392f2879dbec72ca60e10e27bcd85f9f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/e417d29dfe394b9d1eb3fb28ff5ca1a5c0ade2bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/4c87bc31e9bd0b143922a98582aa1619c670cb55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/6f08521737db482fa5aa0c480553ac0892b860e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/72227881a4b0c77d511ed2369fbd98547280aeef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/7ce9b9994d141c71a62ddf9c4468ffefb8983ac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/aeeb9625bb1401e776604fab1250b92a1be61a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/d13dc3306f9916588202fef6513a6ee32f13a012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/4e5fbcb425d3ca1a4d201773256a1f6558e4f3a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/90142d06b76c8487f787fb907ed7f34210ed3525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/b5d4a0776c61dbb229c2daad4ec2397bdfa900f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/c4fad19e35610178cbf37ea0594c00e36906d749.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/ce5bd052719bff45927733dd30c4d755c449f0cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/e84467719a41de757c2bd21bf435b1bf720d92ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/2173ddebd453f27c23e9e995aa4cb456dc1085f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/39754b9c49aad100512a914eb072e20ae074a550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/46de8d4749f7569fd2856d8bf4dffc201cffa96c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/89490c017456fbc193df366514eb2e87a076c443.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/9ea856213355ff5e6c24d64ddb428d1aac0c7353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/a4b697dfbb7a8679538d2aa27b28298b68d26342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/e15bef275ba65b80c614722669ee3a48bb17d85f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/08c3be2e94db9e8ace7b92d1124d04c2594b2d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/5cdc8591b19b87c09b972b45dfdcb0d5a8011cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/714843363cb3b66c44b846275add3e36c4f32979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/8e367cc25bed86e28aa171eaa0432ebad76251d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/a281d101dce468e7f03325d2d97b13037c77ecf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/ab24425aad3fcaa1a61ded3c629809e3a137b7eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/e29569116206b792873e3417414a9389d21e6ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/077299c60ebcf9fc9eefc0255e6cfd310ed230a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/0cc16e389b7914e148165086c4c2af76a326987c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/5048d909d8d9fefb7878849ddfa86f0ea2a9d89c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/616a01b9bd9cef9232c2b6c12999aa6cf8fab544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/61dc02bbac90159647634d4a02bb5f1ce9979e39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/b5cf00a1e46b8b6fe15107539c2338440ed06242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/42eb8ef219351552dd6cf8197c05b34e8a91bf72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/8aeae44330d66798361877bc232784621085cd44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/a95972dc20c3c8caa686db86c48737cb67a82613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/aead2c8d0dd88064717e4df0f59dc3b1db5a5d92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/afb654c840f81d0c806512051adc436af2bbe3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/c17434e9a78d857b7db0b734a701f56697b8269f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/2ba785afc8a1bb739e33f4e94e4daac39dbc5e16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/2dab70fe890f6da4d15791aa2ebf2f70df3258a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/8cbaab6f1d5285cb9fa53c1647c14d657260ea1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/98882ea0e548838be33a07e6cfc9d8aafef14fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/b2baa5333702ab4adfccc7a38a70e16c7d81f637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/b5e6f8a82fae0b3a30b1294288529149dbe5269a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/00e32ac931c7c8a2914bb11fe5cd2e6e59a42ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/22d97fd05b544017a5f007ebc6a6b3cdf165ee7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/6bf1d9debbd981b1d58024415c3d2aca5d95622e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/6e4b3db18dea4a9c50fe7c4e763292157eeced61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/76322113d856792908ff1a109ecfbe7d663bdfad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/afa4f6b00ccbd497b3f5bb00ca5ddfa57fc3f688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/b4d35e0655a4d83ef676aefd7f207f9bb97eb6c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/67f3f1064de68fcf7d6063ae71936a8a4fff4f2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/9cdcf430e6cd411c574e6afdc075c6bc04359869.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/9fb074d7a610b398e7babb6e4373883596115a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/a47f4df172c56eac9b5f72edee31ba64887f5a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/d0f930f93b71e63a440c03b10ffd408f6b7046c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/d41b020cd3cd96c3508ae17ef4873f8c586dd4a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/da7dd9d70c7923104d9ddb2536c0db779f3500f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/059281247dd2da738a435d9240035c0e4e540538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/49a76b48e1dc602ce564580469823836d16bc7d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/beb4176787d73fc83c78cc216fedf487c2f01fc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/cd7342d8644f05b29fb06e06c992a963098cecc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/d45ce32e4485cd074aea676496fea8c89637997e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/f222e006ee75d86573554be8d44290f3d7ceadc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/315e44c5378b4902a183da139c989e26b101d9d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/4402c48bebac3aa979d433ab4c359b8efcb76ac1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/78ef1f42397589bb12c0aa070f6d05868bf1ff67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/b7147cfbb66f86466ce3dfc05c5214f5035e10fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/c04d3c9ec023c6d568bb70db6fd3d509f66908ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/c9184d5a2988d7078afb4775c371a828c20ca502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/737d47f30a863d8b491b542ceb6a926b8e7f3f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/896b969d624c1431c212538e80e4853a8fc15725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/a1c7d7b78ae4616dbdf5f789d4ff7ea7f4b89392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/d5aafcfced4e755bcee1d3cba8e9045fe8c4faaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/e60f6b351c0909df1d32412b40d68baa04d87709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/e8add3fc74320913a2445d984771bdd4ea3c8ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/0cb34b659cfa61336fbdd31e53f76d9d00e51ca7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/68b9c7fe5c035af16d3c5799dc7677f100b91ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/7537342caac69147f9ebb33a45dcba86d83d2810.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/81b806e9a8684cc4ab6d637d0a1aaf27ab33c521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/b0334d0abbce28adc5ae3d3a758bd2cd328893db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/d8d238388ae6826fad77edd2a26ffb5b45b3d054.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/eb0789e332f68192b3ca8bc5e1e6cf069dfce12a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1d03a231cbdaef37d8a5b4887c4017d85e9cdaeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1e89ce6d42990fff78c8977f30162829a1f619d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1fd490604c070555a9a7086e94d98d4c41be357c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/244d86f9310c6a971993de7a1d327ea435fe21b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/2606567c2aecd800a5c06da9eda50c928a94862b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/278aafa0c5d2d5a8f850456e19d8260be75887c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/27cbf0035840196e10d0fdc80482931b7eeac2d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/2a341bd9d05a9c1b58571d6b2e04d55cc674ee33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/2d2d743cc2b336ae95b38b163c186a4a50a4bb98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/2e37bb487e18048a7b80c26615ee817e0dda750b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/2fbbe0815a8916af11da41d719cb134279323647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/300d3caa41a4bec2dec042dfd44488ae00c1ac8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/33d47ab0df023e3bc58ab1778784a4ce6bf1bcce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/37c2c1b9bfca00a67d9b61cffa5dfeee5c9bee8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3929ad2649975cabd06ec1cb99ed4dab5753d10c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3a8be0adf5f9d705ed8d838caf24c716457b5e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3bf0b765597d9f229d801b210fabb50728ca4af6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3cbe04b4c47145b9d650d063b364be4d964e9230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3d9512f5a7481030bbc1e3901fdee64e19171047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3e45dfbfb8db790d63a8a707d2778c754a5c07ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/4078ecba2863cd391ecf3fb3a006faba7db08069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/461889869c521de8d021bd13a87b4bdd40053ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/4c96d66c43c895ce0f2997a2502558b7fa917c3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/4eb67d5c2cf22f5a520fba559cc5eb5bd88e588c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/4fb4737d874a2d84d17a991fe70d95843c66cda6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/5b8e04c45467f49ccb13cbfb5bc9207331f854fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/5d4ba44cd1f4fb1f2227398bbbbed17740f38fd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/5e67317b9c5cf1b635b3cdf029abb8da8dd3fde7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/5e937756eb084582cc8a1c0bf84f6229507a0ac9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/664ec0929d8d81be47bdf9e81b02c4fbbd02052f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/68d8aefb95fa8d9d99446eae24ba01415d55f26d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/6b1511dbc9e8d4182b5eb7d24125dc992ac6ec70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/71759731cc56770953c44975c2cda54c7ed588fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7529d4e86dc632c56a511a8b9963d3dae611cbc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7c1333eac588ce9fc45fbeec99720856286fbcc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7cdba6a5e79ef3ccaa606dae4135330aaeb89bd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7d43f9369227252644266adb958ca55e14fc886d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7e1eccda51330dfaab4cc4619cf7b4a87ccc5146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7e3a067f2e03dd7d2c49cb7053e38843051e27f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/821043eee28e1cdef7d9f63a11e72e684feda195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/824bea931fbdafbfe3277e8626e1e005fc61bc64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/828e3ec9a412dbd830c3ffcd53d51fb05733494d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/83a30bac372758da481804d2c63be2b66276b2df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/8a0c98c59ed4995ebf9750b5ed2e3b90d8a4afab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/8d75713547bce72132df7937b13499c5df1e22d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/918b8726782bc472ef997b861e832ea849fa639c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/94671623787038e333382515366c3098c3aa7ee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/967303827efd940065c6535437570c7b8f22192c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/9bc79bbf0d875195820deb762356deae8b246b29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/9dfbbd928ffaf83abe0a5bbb99c01f8023766510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/a27bcf6eddab8621aeebb5bf1f03b5a79758c71c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/a478af6f7e44a34bf717c53091546eba1ce8f770.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/b2cf8ca985bfa57faa2cde91d6b784230496f2bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/be6a3bc5eb56ef76db485af5c9fea76a64fd3e42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/c366dcd6299414f067de07c723c0d0b47671093a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/c6688d41b1f841be0f2f9ed008b832d174d71e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/c7915d98caa45d49fd5a51fd5e8060551373fdf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/c83660ef5fdeefca2363ec764194458b0115eb0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/d1a7384c2cafb6b89daa969a7f5746497d0fec60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/d49c707b555a03495c3efb57a601ddf36e846904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/d834ef9b76d20d7171e1b3a1524241ae75be4e32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/d9387df4038fb28265b0a49efebd99121ffb7fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/db7f1ef8b1c3614f4ced6e8da742deb6f9c8686a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/dd1f3576b776aa615359591cfd6288585623d8a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/e3f5a8eda5608d2b9dbe8a237338b54f717edb93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/e513560f65243b870d06ddfed2e58126dc5e7471.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/e9e204769aa305722a63ea0c3d37932bc38632d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/eb82f7254b8623f0e1c900af5829aa2eeccee60a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/ed19174c4defc8527283c42e087a030132c0b60e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/ee1d3049f82f65dcf26fb6d8ee0f32cf5dde306f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/ee291b430ea063d4bce568e4b9fc849867ed36f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/ef49ef76497baa33363b96fe5e794d392d4c3314.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/f233897669d962f68280380797a555235f46c99c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/fb5eb1808540b09a37255cb98542511e05f91cdc.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2021-43/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2021-43/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2021-43/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2021-49.yaml b/configs/cccc/dedupe-url/CC-MAIN-2021-49.yaml new file mode 100644 index 00000000..7f68a623 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2021-49.yaml @@ -0,0 +1,661 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/013daa0e9c09c96aa01c6d981ec8263610cd015b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/0889faa4cdee10141689d23582a52b41e26b0966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/08c3e17634929f3795a26f0c85866266982b60d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/09955fe9fb8ed7e4b5e9d31a643926942811cf19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/0a192a050cafc5433714870b597511f64b079662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1142ffc83a6ff01f4e482c5d1dcf05be62d8d0c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/137f6153429106ea405b0b233dad0c420d152a4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/14d20652ab87f8da7cf889cd65275b21f810d390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/153aa2f80b5a049ae1234b45b0dfb8b6bd50d10e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/2f0b73a1bf1b2933eb056cde80f567263f197a4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/6faab736f9cf749e91b88df1e3ea81f92424dccf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/ba1c64193bc2d4cd217d2a7bc94068588a0221df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/c09b9d0dcf542d9b439e4065dd57a415bb56499a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/cb5ba5222f2a904aba26f0c7455d4d1ad7c9ab26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/fd2c34494d8c573e68669f4031e0017f5c413572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358078.2/warc/1d34b4fe2938a2e1e12cd18ed2812fd75dd24287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358078.2/warc/1df1bc7329d307280aaac86d3c0949a45071a272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358078.2/warc/2e77e6466a25aa95387123b45ffa18da91a61d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358078.2/warc/7cb64e07c9d63b533955a029871d419589e6c553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358078.2/warc/951fa1d0e48129aa9de9305f169cf87f28c014f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/26d4887af3236d8ca9e07e578a5882e3d9c09e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/86c56eb504805a41ba1f563685ad5ad6894e41f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/93baa164c308826359e6bf785dee0d12941c1aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/93d8cb33e1c643b33a481a7ecf94d42ac45bbee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/eed9453cf1fbf9d500f717a97fc5dcc3136a4d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/f3e35e10d15a1e480102f5299408e6221462bece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358153.33/warc/0164f2168357d33bfe8187fb356e6aa58bd02a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358153.33/warc/0a07b29333691b0d2db42d911a71da8070145b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358153.33/warc/2fbe4e0735d89040d93de3d8c17cf6560d956b49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358153.33/warc/63cbcce00f798fcdcbdd49fca4878e0e75c489df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358153.33/warc/b974be69f35f021152cccd9057717067e284a775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/0ffbdc033767a01999c18a2e303732116bd6df59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/1d89f9e462f2b17ff3b877c80397c768a36672f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/230173b87b460f1a8a9b838d9ab546d30b1f083a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/4ffa51db05ad3933a10066fec034c26b65b2bd45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/56f47020f93bf09f3ff53fd41ebfd01ad4d31fd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/d458ae7029059314257b113f9626c6110079db0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/71bc51544b257b9add9668d10a31a970595b7dcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/7fc1d4c4ceac78977404c1b8d6fb733887d1683b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/e094e7d67eb20e4a0881b960202310828a701a4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/e74bd5f4e47f62d4bc830f1fdeea534801877d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/e8b65d0a4acab270f3acd09a4b3d402645dacb58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/f5bd3ba422f78a6bb61271b151728968a0bed87c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358208.31/warc/08deeb1d0882acb9c37a53856cf9facb924c8175.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358208.31/warc/1d1653bb4dbf7cf9b04409759e11642f662a87f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358208.31/warc/5f10c57fedcd2b84dffff2ae84b61c853e8bd5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358208.31/warc/bb9fa9b2a7055aa4971e1b151f1270b276b3b2d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358208.31/warc/d7f08ddf960602e42589f505e960911f2b41f046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/4e16d14ab1d7c95088b5d3452608bd35214fea85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/94048e98233358057b21522e6a07f6ae0676a3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/9e7c625a32782ed88191c7c562e2ff45f83d5c27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/adb7a4f65c6894316dcd45c1e07c1ea3cb97e091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/afffe83f2dd3ac7f1b191c527a76d2584449dd9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/c27a67ddf66cf8901fe33488acd3fc505b43b02b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358323.91/warc/4dd2eb619d050a298d1ba1fcda83200279e4c3bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358323.91/warc/809d11133e26ff5243f4371803e5966079ad817d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358323.91/warc/80acde6722202e323334c5095080b800e48f4098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358323.91/warc/87df75e87becc20af427c82b9f00bc652cbbd540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358323.91/warc/f67021cbd5e0c08f1ef347556b6df933d076e9b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/078d55ff60417f0da28d20475ec4652eb29fdd26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/3d20de8293e59189551fcde2314c12cd29260012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/8f87ebe3f884c65e782a53857c1972893c5009fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/b79e6141bcfe78e77aeb175a23b65c33a15d8a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/bd285adceed809e3913cb5b79715111e4047584d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/c92e9307149bc2234ac3e43916cc1dc9548bb234.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/06ae7cc6905cef0731bc1dbc2de540e88b0e8585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/3b175712064c79449b8e2c426a323c9372f35d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/3bcf62af383fe3d70988172e4d06bea57cddc92a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/594146410bf2ca4acfae6a78ae3b26f53746530d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/b2853ddea2001db265e60fb501e0aa0ca35d6722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/bbb954a03426d36b400887a95fab9a3106bb07a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358480.10/warc/43ddf2bb1be58383885f82923a8684db59aa0798.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358480.10/warc/46dc4108f80b5b806cfcb8b623fc44c36e34ca71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358480.10/warc/bf6cf4461b840f56428d3067cb43818f465ab122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358480.10/warc/c847323cd1dabead98d4374f096abe0ca732b282.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358480.10/warc/c955563df9aff93b1628e68ca8040e6066a681c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/195a6b4833ebae98bedf749c20fb0cc09aa68e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/484b5890ec131024ada89a1a1bd03a25af3ba860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/969c26989cf68bbc04646341365031f28a92c7c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/b48370611351550f55ef8217eece09eff0f63ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/ccfba2fa3556825ee37dad9bc76d66092feea349.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/e974f52fbeb537cc806aa941d4945a9183db86e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358560.75/warc/1c020719ecb4ee33b0365e2ec8ec784ecae1ce94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358560.75/warc/27ba3cab20c16b964bf622abd7ad2f5a14ca7023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358560.75/warc/4d2e93520d44375a0d8cc6326385a4e5c285031c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358560.75/warc/75d666d359ae35676b0a2bf229ac86374d11bb5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358560.75/warc/9fd9bcc5dfe2da2dc07bff310b8438eb8f5846db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/47e012673d4a71a05f1b9ff630aaaf236162f747.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/8d1bd70e3605f26ab9660c2a152a90be86a01a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/9624cc6aa1926fc702243ebdd2029265be3b98d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/962dcf60f9f90a8dfceebb2c9a2ff0a645b4d75a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/cdfabfac469af26db3f18d3776644a94a9ea65cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/d8a6e304fb4b9ec3f38f3b0329d6eb3838d33c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/273f2457b2fc030ab72ea3a56c97dd09ae842a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/768efd7282b75dab228ade7a2c7715a304bf8de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/7abd866e9fc66437add96bbf5bb7033c3ccc403e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/851321c36f3aff552a9defc173b36ea68b0b966e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/9aef5284b0ad35bd081c2fa4bcf75a1433ad508b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/a839bd814184f5879cb2f1873cec41f0d51ca00c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358673.74/warc/2865e792757977246708eaae9faa9e316b38d9ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358673.74/warc/572949264e248fda4404b4a59ecaa9b0a9e62079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358673.74/warc/649d86c9a21a73ffe84f64a30f8aa20efe866308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358673.74/warc/c6820988f66073dc821a26895e18a45c663de322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358673.74/warc/e3ca7f294c80ae4cf960a7b45d24d2dc3fcbfc78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/63149733652d18db1d67fce63906efe93a60d7af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/7c12444821fca84fbc6bde0a448f7e4c6657a852.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/96005d3b66b2220028b71abdaeaedc09001d853f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/ba392433aa59dd383dc318fd3e52ba98688ef95c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/dee8570485c20b03f06bc8242986aff286f7dd31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/eabf4c575d9b048566b6fddc1d0b718da63d6a87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358688.35/warc/23b5de818156412bb523317f645f6275d5f93aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358688.35/warc/61a4b898e0dca00786685249ffb9d399e17bb769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358688.35/warc/8b2be4521223b0d0c20e2b4994a5547c3337399b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358688.35/warc/9da31e1d914194db2299675d89b4426b01f103ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358688.35/warc/c2cfffde43eb5e9af7d283b4d20e0bc9109f5a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/10f4d2382f18e2bb7d17fd658e1abe3bf9d6adf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/3ec0359b3d0b9872dba2d54048c860330bcdc3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/73bbaba304401dcd3c23617a8c7a8f9829aa6c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/c388d2a171f530ba3acee3c8d0b4a1ebbafb7942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/d57beb1831ff00c7277948f5dc5e4689a4c84fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/e77bbe51c42e081562d2ceee3623584e6cd62f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/5fce9f41c372ea61ee84b676aad92d5d0b596865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/9132aefd2708c5f75ddf470c500f74b8920dcb2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/9dc2493c5beb2f66cde3ef48b406a3afff151c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/a3eb4191a5e7e0a75092a847f96309b17b43cc36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/a50def8c23a668015ff34423cabaeb849f5c7fa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/a5880c65e135a445162516bfa9b0834173b1bfc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358774.44/warc/0e251c334765efc6369e133329d2e08e143858eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358774.44/warc/aa3dd993ae4b8961066a01824ef852857e4c9ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358774.44/warc/c4a897e8c60f541b61002a344433e32a5a923fda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358774.44/warc/f2afc9af456e83a592c7a7ed26e2f77d92b9f370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358774.44/warc/fc83d1e9d0fc8cd65a1e8b15abc1f14c8ff6a4ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/0e69d22d798dc7a9bb310d2d165a3bfdcf532b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/0f27cf29ec16a6e9a8ad1bcc64f7a7d8584c004a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/1caa0bd1450c3c43f40ea3f993ea7ff2b7423972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/2d4fb534805765cffc466fb785dc5b6e3cd8c0cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/65fa2035ba1675e85513fab18b21630ff477e377.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/c39028aacd4ed347e21916346ce56bc26eb1c4c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358842.4/warc/14fb8d8f00cb9790f5d12d82d1ed878a605e8070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358842.4/warc/4a896afc1490e3b65827273c6e4293f1a53c0031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358842.4/warc/751b79aab17ed89eaef958ed515801438f2b23a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358842.4/warc/cb8cb03ea4b7cbc7007885ce8fc76471112a9728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358842.4/warc/f611068a395abd1bfac99ea0e72919638246e20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/097aec118e38be7d3dba1e589aa9b5da34815300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/21886f4fd9638e748f026c4c476eca5f21c895e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/388e0e9252f073910bdf75823d9ec7e6087eab49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/56f84998efd8c06597b9eefc1aad682dd065b609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/69b41130a8697e7ab28136d20d5e1a3fbb107847.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/bb0b62051bf0290defb71ef4cd219735ab6703cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/2b65956c08e6184eddaae38e393dce52f63ba1cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/366d1febd932c4ec5c0e6f96a76bc0be6811595c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/b9134db1485a2fb97d6b6abf8d8b604334b369d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/e016a2a99fdc47d7c6da2e64e2c7cb5a2d79b120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/e31361145604938379618bb16b7b1676be440cff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/e87598084d09777a63b7ce5e308297eb30c9c560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358953.29/warc/1c05050e347972beec8af2d1185f9fb8dedd2214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358953.29/warc/859a37f406df5226b2eaa863b8987a56162c153a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358953.29/warc/986c3c4128e082fb0f2d82523c491ed35cd5280a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358953.29/warc/b27fbecbcff0d08b5861a42316ffc0cf99958521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358953.29/warc/d48f1fbbdce9b337354f4233487cde4c49dee872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/460d38008e848bb1ff75460cb5ac5b89cb24904a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/5ff21f56681dc2ecf94724543368a8ea28d47da8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/90c077bfc4b7c8025ad1d11162c6763f4032c0fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/a1ea182dd71b95a1108335776769c44e03c5a0ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/bd25184f408a714d3b3935547bd1a1e05aee0d68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/cd84641dd1a8f9ead98b59bb119a41b0ed0dc6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358973.70/warc/16d381bfe3b8ccb6a8d5c6e5523f11413b668fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358973.70/warc/3d4359425c193ecb972743f42c5f8ddac860f78e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358973.70/warc/6c8de340dd722a559dff78c84a1f1cf48ea8249d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358973.70/warc/bcad9d72ec45db68c7039ad48ceb3c54a3f2dabb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358973.70/warc/d6629f01a56b5fb3fb64eba3475b35055743626d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/37f83e47d1c18de81f09e80c02e341b41d5887d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/88f6341b6b00e114a23c9ce949ffa986bb3a6075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/b6bcc526d28dc1e26ede6cd3778c5ac899fd6e6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/cf0eb9eaa678ef090452e2c26fc1bd448b71fcc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/d1a98fa06f6ec5173e6c266c36799e6aaa7da839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/f571cdb1c22933151a736db276351146509224a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/11431406e47aad3c2e0943161ec2bbf224a37602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/2a0196d427f402b9001fc14ca1593dd274e08117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/755b8b8d7811638409998627fd943849ab72b1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/d718251a89e1628ffa26c64025cf8484ce2c91c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/f7da050c60a93b99955e4ae29cfd5cf6477a61d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/ff9968c9c791f1373c27f155b38023a769ec9acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359073.63/warc/5e9b372cf1d8884096c3143e0a54387bc1309c4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359073.63/warc/6276cbcbdc092939496e5bed9ef3601475ec9d17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359073.63/warc/c08e0c6cf0dc767dc7b2a4f8bdf150c490282f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359073.63/warc/cf22064c2b0241170087f9e48aade46d79eb18b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359073.63/warc/e79d58cebad4dc2f5818e8dd138aeff5da0f925c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/0eb3209a086b13866b5a7ef1539cf9d5bd8ff0b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/13f935454423df8c3185d6fa61e958ce8eadd4e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/290e2493a6f50867b5e9c9bb57c10de5d4e7a365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/6794a2e260ff832de4c54c2ccd55a8f707479e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/6b0c7db4f2ef7aeecdbe8b1ae9ea8abcb024c6ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/cd13f5902d663a10c714e6c0f301968211f68513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.78/warc/5cf24defa24fc343d911db09d5820d4aa6944ae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.78/warc/5f64f0ce7386916adfa8d80aeea6f3d3c3d59b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.78/warc/87eafb875bd554c18aa8dc0d83e0c04a25ad0757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.78/warc/925c91febc9af8b5b6a446aeea8a8dab7f20be54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.78/warc/e71f8193f0e91c59da95ee160d0dec96b6812146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/1e65fbc36cc77d7944a1856f3b61c38208bb9ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/c829bc7548ce3562d73748933aee15e7ff7a4a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/cbdbe748d3462d85db9bbc5ddc241ea45b0042c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/e4098c8848df0a9fb60f111369248bf970b502c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/ea0f3594f8e31154db14786f198e0667ef93ffda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/f3f54ad584d0fd7bcabf341045caa89d82e01148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/0d670428fd18e0443ba13583599153be6ae50cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/368f5bff949052583ba17b51381beef642058645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/551729813b5f033a0f7ddd9ed2010fd5d9a39116.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/b0580c2977f937818a5e558af82f11aad103d407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/c810cf1cb0614cbefa31a32aeb39f785eefd8c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/ee4964b3ee294ca531a4ecbcdfe3481410fa63ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.0/warc/094d890384393ec92d1b39bb2061cd501d8c19c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.0/warc/1d8e3ad83b2678c0940fb1cf1449d4fe443fd2ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.0/warc/64982f48d01c9a91bbec166a3a851e8e9400fdc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.0/warc/6a6da2763b48926b2dcfa75eecece135cd56c5fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.0/warc/bd4fd19576826cef9e2b87bec4bb2941320fa8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/185bcd51d033efe04230b450e462cc3e3c7f97e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/7bcd719fb5f9814af9a8f035af7418274e831cd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/89f51d0ef3cda54dad38e2e66e7742c417416adc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/a7701102e7b45f5cfa3e21703e47f675cab6c1ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/c95d6b3c65611e0d40864b413912598fc518e202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/d3993acb53b8eb0f6cec457a6b3cbc89a45440ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360881.12/warc/51c62d67f3b079a733b1eb21e0e285e36c92ff36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360881.12/warc/5b6e03e5526cb5d3343f803aa8b10ee161998cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360881.12/warc/7cb6b4f67c7ca2504af3c2c4830e149e77d7356f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360881.12/warc/811566875c8513ac2e3431837ece24a5b7e15313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360881.12/warc/bb6462f52c8a7e32d6197772654cb478846806c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/318b6801b44098c548c992c5a3b581b05a45401d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/461a83c6f46cd9062e362bfdf0660d2a3fea54ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/49b4503c43bffebe1c94796f2ef6422dea928f62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/83a4609e2652132fb5466d1cc1273f8dcf10bd53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/8c6de76801f39a53f18691e3998c7932d1f362f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/e75964bba962a699fdd0f3ecc064387f39693ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/0f815cc4fdb7dd43bf6355f0fbe11a071a52ab7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/43585a3ca0ee6d64b72b87c3b71b2352cf89f7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/9f1e13550574e0f6505201fd92d407559e9d8cda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/a0338b224c12a15d2173c86fce8c6168548e0f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/bd3940a92be256ac060d3afb3a774a320153f392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/c57960abfd693e8bf6693045175d0dfaab06b9ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.69/warc/3aab2c2d758f17b37f9c17aa256e86dd6828b3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.69/warc/473d73c36d4711622380c35d4694f5fde9fcfbce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.69/warc/4c692d711da95037f319c022b8706d8a36c8ec50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.69/warc/d1a4759bfc1a1caeaf449be53b4d4c5ff8aaf815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.69/warc/ee1e14c106dc1f6fe296cf4c3b47619c3ebbcd9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/1c9d496514cf07012fcf4d0d61c1aa2a695adf32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/2098af34a9864a85691827a163f8dba4f3a7d3e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/2ffb7a118c93d90081e28219320df05603b978d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/4ff0697a78f0f88300e06ca107a84550a945b62e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/9a3a232da6252c50d31738e5872cf518b32f23a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/bf30ca0f6b9752be74de666729ea3b4dffbadfe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361253.38/warc/41958c46b421f3536737b0e91f209263a0531d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361253.38/warc/5c7fee88ad6b04ec1adb2b538512f481e3e3429e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361253.38/warc/91f2a000a4b910409c1374d03d5c23403ab34987.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361253.38/warc/9cf450fa1d05c2f478d2a262445164c58567d05b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361253.38/warc/a0ae4f143d6b28a6ff32770d6b7b095425a07867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/02e1cc2e2ed658f61f483d05975a94a17a3758ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/3475c7296de69e5f2a556d9b85b1e23a201c1bec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/384764d65e1a389ad0536cc455dd14d7909c385b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/48fe8fb1cdaca998625cd5e1c5da7f6e336c01fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/d630ed49b023364f4b598b7d73e4593f9579863f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/d6e2945174fb2822669bc5c8097c84808d56f81c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/0e7b20ae5c81ddb24f585a99818a531954d41808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/5daf54e326388f22db07f2325a00a6027875312a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/ac1a069dfca5d520dcd98f8633f877dc03035454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/b21d2dc72e9c970adc650c4af6fafe3d8f378de7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/bd13ba040e5b322e82875fe04619745a368eb1d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/bf763a335d0d9e5eb32f91d82662bef78765212c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362287.26/warc/44844807df8ba443b1138ba6eeebcc9af6da3158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362287.26/warc/babbc214d3cdbf5cacee74637db1c19f2e4c8fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362287.26/warc/ddf26b43d97c613935a295683c40e68edd6b0246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362287.26/warc/df80f303b25de94e8667cfb0efd92432ccd2b77b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362287.26/warc/eceeba37e6993b95e981900ae66516e95e31f665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/3c57969740938e6c9dd1c321e24f3d52fdc33ae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/4787d699dca9bec08a30ddea0fd4ed4f90497934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/710e733ae034443dbb7b5ca6e95cf4b4283b17ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/9910a7647a4252e056e00b092fb0f75d940c62fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/a7b25695fcd5edf695707a5819063a3d0a54dc16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/c4a452e3fccd699e6b88ebc00b90f52aa3dd5cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362571.17/warc/081f4ec92b6dfd6d4c5263698ec41f6226c9b114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362571.17/warc/43655e6ed7616d4249bf2adb0782c290dd4a1e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362571.17/warc/65d2d4ed1fac3bed3cd521e1a73ddd4e6416d80b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362571.17/warc/cd78fb15d7418788e4365f895825db541834a381.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362571.17/warc/f8d1ec6b4e11c683d7478f6ea8fedcf3a3ffe57f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/25be20cd14174979b2b498fec6b168a2780898af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/691def238d1d29c997deebbc24742943dcdeebb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/a8e88cac187314a66d182668c76efc6cb35f5626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/ab23dcb52b7db51b01249aa1c8407bab655bde35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/b01570d59c581164b9f5a16817b667dac637e313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/de78f1b7124498668c3893f69d68c51299bf138f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/52ce3999bbb941c9f92eb6b5688eb32933f77ee0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/6231e47c819b8c55e390ac166469f803fff7cbf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/84ca75c7c406ad0e15a4c39a04d126015368b9e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/92c04b988e80581f265ab4c32e086ef08e974d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/aebddc1953c3ff8f86a39b80408ba5a9dee1ab4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/d961c72c0489c44a2bb65982882742320b18bfe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362619.23/warc/325f88089d0f4db9f8fcae089093f9493dd4e7d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362619.23/warc/534eccf7154749383f69a1e0696a41111576ab84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362619.23/warc/70d5856aa3815afd9035f03ae86ad60c284096f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362619.23/warc/8d1f622c3d70b002481df6967a5fe893de2aab98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362619.23/warc/e4e6189a26f81a9956ab22d3855899398c7e1cd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/1153ca4a7abab59887ee7fbb8c92298ff6df5a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/2a1262a829f0c42dd9fd39410a06dae30ed73e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/6d17e53c96a0c44785480795dff22aa2e4a45c9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/77ed85013288bc7d280136815b48db3486f8f44e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/fa2b1db6590aaf14b889799809f3d7bd3d50451c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/fbe5706898529b618fb19e78fa541b58d3b49584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362891.54/warc/3b95350e562dcc239f1ee7be29585973970048f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362891.54/warc/4296db88620aaab8915e998531b9bbfc8a2a1924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362891.54/warc/9760f36fc89a1c13435bffc85a1e609f5f7b085c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362891.54/warc/9e5d0eeb74e151c345029968b08432d406196c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362891.54/warc/a94794d0bb7ffc6d53c2203e832d8f4a03f7b6b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/137001a15b88d16cc7f84c13ca43d0f33a6d466e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/3c4b386773c2ca04faf9e3fc17c3926a7ca827b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/4c253cc57309a720d4efda0fb8dc3bae591c2547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/57ab7743b5292cdb2df0cbcdc9d3fa8fc301747e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/5ef54cce18f4890d664effe798523eef1872d253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/e2da8f3843749b915db360f99bc4b4e846deecfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/3cb6d242be38eb570abd703c775c9e74f38669d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/3f46056ea2d5d3696e8dd662e3ee9fc2192615dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/84e3a190afcad5b421847f4c42b23fc73876349e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/8782048d60daba06c1ce776d36c75fb6ed968368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/dc4740f6b36d76093a2c760c2126a6a6aee63c7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/f884073d40c73aeeebe2ae16afcaba8375511620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362923.11/warc/1840bdeaa8b2e26d680fb88d96b222b005f6cee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362923.11/warc/19198d542ba3c5beb0bfe6c402a06279f9251585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362923.11/warc/6974a1308e33f415c5dbadf939d9a0653df77f1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362923.11/warc/70b4c6e456f08e1469fcd6c767155ea8ee1eb500.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362923.11/warc/834311aa70a816f433c5e45d841053e819955e5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/2c28a3fd675e11d7bd48f32f190ad8d9cf3eb662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/6bf0ed291483cb4a2cad7d566b0589745df9c2f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/9758e390928456a967d51fda401c59e0b4967dc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/c03a06ead5be81db22562ba680bda6613e7f07a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/c1a180325da571e4054bf6e185847b3d926c71b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/ff695f69fe014811934d6f6c7c34d8713794779b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362952.24/warc/300514555ecdbfd9e1d7e94366d111e586b62ed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362952.24/warc/7fe9e34433052518b85c348adcd30da53a7e6b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362952.24/warc/80a758b4bd64b83ec2acfb8e7f8f1eb9c92acbf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362952.24/warc/8b81d71e68d048134d3004e0c816e6643ce9a159.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362952.24/warc/be8bc63e365ddbf450bdbc6bcc73710563b276c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/1279d3b9684146cdebcdf112a9b9aa2acb1f89d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/1300f400e586dd68859ca633184d3942c72a66f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/302f0337cc7ac1aedb40f9e773dc4d54978e571b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/3958d9318940771722cf04332091a68a7d8dc225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/84f543fcabb67702b699ed1dc6684d656dee8549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/c238fe287e757f80a1d87463dd4dd1c2ea725aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/172cc3748d21a18b9557155ba3496224a3d27186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/24139bef969ced2da2ce782aadc554d2adb1dc01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/250411745008e87ed6b4166d619f69288b062c47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/91c2983105280963be6c05732830d29ba7a6b639.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/d6da9a1c450405c6c79cfb1adc8432434ab72056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/f2445967af7a21f43bf0fa797f96cc5a3d5c68e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362999.66/warc/1baaadc4d71920f0bc6b6ebd06023f02a1d39011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362999.66/warc/7e438dda3edfad6cc8db21a31847aed10690f31d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362999.66/warc/82cd9749ae06858930bf5dcaa6653845d5c9bfde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362999.66/warc/a88e70bd6dcf27b374682213914c50f2626b2ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362999.66/warc/dd4a2e4dc144e35c703c0bc30ca4041be2b9ee64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/46402b9545c325ef0cfbce5bec38241818bb3ced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/5f8d541b262a9dd767f611e36b3b67d91c04f951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/af6e1071fb4bd2a6f0d6c4a2960af4f986e81e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/c361534f35308d6c868a105b6e856479f0348df2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/d703fc182c1b665bf77e59e008a1ad8b510b55c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/e374171a2fb4c6ca1365d95527f221aa03e82052.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363125.46/warc/0713752771e13fed0d3a71b59c6ec51587b204b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363125.46/warc/a156aafc0be85ffb9bb4cf745d87b95576c1ec87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363125.46/warc/ac751f86b91db422984bd1d6b6c7cefef9785714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363125.46/warc/c46ff5df76a8c8d6bc876e9edefafab80ec09088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363125.46/warc/f3a0807e37c284a853d2ecf2903d1ce09d09d7f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/2a02b2c40447b07cd983f864ddb6ec4570312ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/4cae4ce59964ad23a492e4b7efdd6b7de6fd776d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/ae3a78594cd96999e5d6dfc1b19d7736a6dae5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/eda8016e26cdab2faffadda6e37d186ca66bd0d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/f3776fa5fbb630714abb84f7827042382ea2b261.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/f971cd3f4d10397bb9d813ceb45a0df92eb7871d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/35ee1c8446c852e79c42710b685c4bb9df9efdef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/6bd12a27f21b1d53135a1786b06118cbd10f9ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/6e19105c3e2c951801952d3635fd7d4757a14041.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/89936892ed7bb26664b43caa23a0836a2f73dc1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/b90a03bb726de847d7f0dc08e2a983aeceff7c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/f1f034d83a5c6d3c53401cf5341320e68d27a8cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363149.85/warc/04b89a452b0bb870f385a9fdac245ce536125361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363149.85/warc/1927533f48a5a8c89af1184ec18339c7b71e01e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363149.85/warc/63f6074d792a04192e8d05a2cd254ec20e196e53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363149.85/warc/b43e60de903ea3c20fc37cf4c23504c2dca78751.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363149.85/warc/f702aceee8af8692f2dd00f99603bf47e285c05c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/3ac8c23f789eba4ba8a04d152c7f120811b380ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/6166c148c8867d5067c2e8cb290e5726de5c9311.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/857edcc72c634d281d5397133e0da80633ed15bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/86a71c02290bbd90b560468fc9b75d31659a75f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/de756f0c09f358778c10b28f98fc47bce1f91619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/fac8dcddc4228423fb2764c2f63f62b93942b39d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363189.92/warc/3eedd9aa30b387cd6aaec5b06d4611eb69dad7d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363189.92/warc/6cb248495efa506a50ad10807170ad8f4ccc12c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363189.92/warc/79d800336ee379520d0b66b9443543b20f630f17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363189.92/warc/7a303cb98651140cecc721672787299d7af33c40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363189.92/warc/8f7eb9a6a55acd6a356219fdcf126a7ad1997553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/27114c9b26737b7602f4418f8ea7a9b582df6a7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/5492e929067c02a320d78db959f3031511b1158c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/6a65ef8b27b2ab188173cfc153eb5edc88dacce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/81a7b917504e13ff69848dc2aad57106de819e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/b6c304c8e771a2259c69135ddc55d540f7cf2b9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/d28da39297bd4461430a9ad84741a3ae0cb39a80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/078c5fe4b0e8ebfb5c5da2297329b16fd43834cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/1bdd729da73867bb65581bdc093753988ebb4cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/3af60a1d048ea1d631db7cc0aadf39bb0662d97a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/aedd3d2c0ab91e43c536dc7eb9e3dceccbeb7e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/d42fe4f5e87438c0ea5bb4f6b8b589a0c6e13513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/d460b22e27d67bf14d1e8790807c7f675d1935dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363226.68/warc/06cfeac3edea862e5268873e90641f13e52124cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363226.68/warc/419e4544a793c169fb77c3604b0b856ece5dc533.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363226.68/warc/7d90bbd9ee32a1e4ba971153435fb20f0d1c355c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363226.68/warc/d7553b5e7086fbc28725d334926469a7f0396c9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363226.68/warc/e15ebb6ae76c06e098ca8c13b2b1905d0dad0a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/2f9214b7e4cd243373c034ee103a174ba668488d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/356ab4f2035d526ef88de48ddc510185c370d60e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/4a52f343a3234d86ecc58d3692e93a1f3eb71b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/631b90a8f66fcacd71af0bd5b54afea741037525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/cdb795bd832a43044fb8d64073c995ad7097cea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/ec433753ea69ea868f69812bcf001541a16e20d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.39/warc/0670840c24631484183dab04860d7313d96166b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.39/warc/0bc0b25b039bcd56a4dce94752ebb93fbd511a67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.39/warc/0c455606e84c2c0d29fe211275e2bd14652d08db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.39/warc/14a9f0f78273830450bb930c00702cb88c8e65ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.39/warc/15081e3c5506e5884eeb93ec954b2f1c61cc6596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/057b9717682f265deb493cd31c474a80c1fad20e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/263b4499f0c4d57fef6570412a8c58a78165f3c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/32f5de5422446e85dfb87ce68391fb241de689e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/5762fb1f696679e97374e7f99c0d421e30a944a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/6653bb974a636a7b05dcd318040ee0b70a9545e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/a7e68807411c08047bcb942015cd5500db6093f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/2eee4ac8cc0a87796d7745b577d39e9d3b5b75c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/3e0230f393117bdc2c2fc14da1c9edbfb975068c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/61249dbc017954498ef0a703e85b1d840302fd83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/6888ef7d834b4fd653ebcb227d93003d435ff8c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/6aacf90b1e530c06ec322a5c115c075c70a6a7af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/fa0417f9b0878751f2ca24aa4c54927f992b28f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363301.3/warc/209d88617b6e47c375d0c968f786dddc9f4949d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363301.3/warc/68a24e41a9cbaa11ffdc5abc23ef31d4bf538491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363301.3/warc/7a8437dcb8c3eb2444ca5203888cbf85cc0905ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363301.3/warc/a65f25890673b07e79252f40c5586995b92e1db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363301.3/warc/fef792c9b23cc735b8f7dd21dd6c1f1e62c8640e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/29146dd34e357b3652743dcafdb9e044e1209cb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/59bb915c0b115e356c5217aaad2f54716d3efb2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/9faf07967a369d906382f79f2eb32c2fd832c87e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/be3973c4382decbb62da5075c753dea1cb9509e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/d2604686d6c3b19b98cff606c5442d9d06f26159.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/d8375c5a58ebddee1f0112cec12b9e8e78cb2355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363312.79/warc/9d553170d430a60b6939c36050903ade536dd20c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363312.79/warc/a2d1d61bc27484165114e2dff62c27c5b8bcadf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363312.79/warc/cee846053204f8b1aeadbc50d87a232906f8ee01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363312.79/warc/e0c0ec788b3dd691f829da4c5a1917cba1113067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363312.79/warc/e345dd3543ceee7453bde5a80678dd4ebffb042b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/50db66957119ab5cf84f4a2405991903c83e77bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/633e3024f16288d7aacb5b9b873d504cd02d32e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/999f26fc260d7f17f5efa8264beed3640a7af4fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/c10971404100e7f02a980e2416c5ca78ab665e53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/cd1702daf8bcbbba42a0a631c81ca93b1ce3bdef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/f99a80c61f40b10fdfbac602cb094ae490158af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/0d008e6ccd5194856ad3a45d70551c68133d6d7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/0e6193715fc70ae9932324039897159a717cfcff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/b94b1c13cf5d27811c9d3cd438c15d0240e2036f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/c6c4c3eb6ca47e3836d2e6e7a8c7649dd2c3620c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/d2c6338acad74f8ff5fe01096d5a78af32d2bb36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/eff81fca201496ec2c0c00f7eb81e73215d1c90f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363336.93/warc/5798c6d46edf25e90b1bed96c3f8dc85dbf47202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363336.93/warc/9cb85b21c2a1e36b7982ab796bae2b13c8681945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363336.93/warc/b2bed539311edc7b04cbd65de2d0914dc4a62bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363336.93/warc/bdaea854be215bebf0ba8cd6e883205c49501f68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363336.93/warc/c0f6488487d3bd52584386d58b8142b91553d049.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/4e0e085028fc0551c80c99fc62b7a7460163e339.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/76c3fd93852b7cbc982b2c669fe7705f13c89fb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/779926ffe3eea1653106a1c97dbaed25ef3fd864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/89925569d21546dbf0b0f67093e8c99b792794b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/9debc4db40592e7a6479b92388305295ee060c5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/a3ce2a9e5cf2bf60e75a5304bdac7bd24e78852d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363376.49/warc/0622e8aee1723c6544d8f06a394d92a6c734b062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363376.49/warc/2e22b16b6135d8a6c0f8aa22056c7f18d95d200f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363376.49/warc/2f427d0c937b3cf58c081b5016f0063cfa80df34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363376.49/warc/c3bbe920ff93536411367286ec4a27acb53d2d06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363376.49/warc/e0eb5bb83777aa1116c013278aa31ad834b29fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/116fa340be9cad00d84a855878a7c5651d34a236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/413c94f9563e9a8e6acc3424d88bcb2d6a303d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/52c16b02a972fe1e5074449ca40fba02c4957f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/8a437c59875f0a50641d76b326fa8088a3d77587.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/9380c7cf4da3c1deb38db5e2cb52f1572fb4590b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/940d36c1386d149000f44d4bfb8b5de546432e78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/14a53d69c48355d2eac21c162796a1e691448877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/2ed2783ef0efcf5ae7358a116fac0351a551fdca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/46554b0ac8e1bfca4345a01ddef0de95c8e684eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/93767ac225b98f0db2dba56b6f00a8be093ef52d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/988cedd2da393422b83a0776bbb0fb03f4681b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/be0e4b948a16bfd3ae10ce4a8936c70e361fc80b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363418.83/warc/470b421bb7542df5e1a75881c802f544259b3c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363418.83/warc/8020b0a05a75e743b1d4340e6673eb8f4c867073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363418.83/warc/ae7ad24f3105d9fb3826cf7c85009b22791f3fda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363418.83/warc/be0c4e2da5880d403c48ef905818f7ac6aae1a82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363418.83/warc/e39abfe7945f8870cfc8035c7b449afbbc27015c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/264f7f8ae4cb452f2d440a2079713a41febe1af8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/5b67f17b61666b8e8b9f50193c3af9b96e963c19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/7ae239f26e9174b09d8e93cea698e760e1b35623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/c4ecc37d52f6a0a2bff17ca31a79943852c7fd8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/e0a43401b2bb64c623d909debc56cc95828b494c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/eb398e14471adfb9e148ad14385df0a9d5d88bc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363437.15/warc/1377e34628f4a036cbccabff02c588706dd330dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363437.15/warc/1bc314b2592ce95db7864c7a5e70a95dc81ac850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363437.15/warc/45026e6273ec8921776d74c2969620de45d7200c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363437.15/warc/9c6b2e29371bab39bf6923fa6ef7841b4c33bb3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363437.15/warc/a0df406fe55bca6324fd32f3830bb4f25f1b04a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/08d820a07406fdaa52f56d870195a2fd85a965e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/10294a2c848aacda94f66bf7a95e0095d0502a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/14ebc6ed680fe2bc47aab4aec814a29e8d4b87f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/43863baa17617d0706b58d0bb74f47d5f310bb56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/b9e439c7af7f0b6e64ea0fc0250067c56be43ea5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/c4015103734c33839b6961128bb5472802f59ce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/595d177b2ad9e924c22a855b19a28a0222aa7767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/74da6c614500d0c19f5e560d56ddd6760c20fb99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/896283171d0c030e661f4f00f8d3c022e4af1692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/9fa1455ebcb38d2f5cf19e7371430e2d1e6196c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/a42b0713156300d706423664131f3c787fe9ae33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/c3425deec965b39d55f750f13c8ab8be20676a3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363510.40/warc/09eb1ec708257fdc17a8a0b901922e346e910e6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363510.40/warc/21bdf1b36dfc9c9b35011853af158061dda78bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363510.40/warc/2ba5efa75a3ebec0e8e85c7f86064669dc4d5509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363510.40/warc/964b6e8cb01e8314aaa57ab06f596439a5c7f83c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363510.40/warc/f91104e0ec4a85c8db5bd3e45aa4cfe37e1ec220.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/09b0597ef648a989d743bd8c0e159f9522ace8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/0e0544e595cf323dea060cc9bc5d75bc6fdfc790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/35bbe66efd45309f1215729a36ae533773a6bdb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/61e39c1b150abc6544f86148d3915534b208cc64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/d5abaa4250cb01e4ef8bd79c425f0d3d29d7ff85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/e298ad24818d49e220c80f3f95deb4a89c62b479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363520.30/warc/6206f000c8d20e69c7d6f2c339398cc17554e12c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363520.30/warc/7109ca8bbe2d09e9eaccdeb0508f0ec78964bc7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363520.30/warc/a43ae006cbe744e5962e556cf47985fb8b72981c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363520.30/warc/ae5e6fd419369684e6bf25fa15299a350d97c4a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363520.30/warc/d6700a250131a6268aea17e2761ac75ea40e4eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/1ce102706a514d4eaf9a70ed661138c7132992a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/3531380f9ce9bccc5ec90cfe0b753949d9761fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/4171c0b0146dde49b36eb3c967a6a0099e85eedf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/5ed7dd02ca3f7a80f964d201ebc468d7fd6afdbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/64f04d9f2a18656272f5d7fd9450380fa3226740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/8b8794483a3c8ed5b18455eb90768c344696dfb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/05669eec0b4c4e9aaf0eb5301b168d50b0d3348c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/287de23a3a6757b327873ba08913cb2f217b36b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/4e0e666390a4fe5297665c0535ab6e63c9ff0008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/9fce26de3c0e45de224c1a96d49fa119e20735c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/fb806f628525595ddd28a6f1a986b82865d62593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/fb817244fb96cb337a8721bed1b14e783c1b9b74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363659.21/warc/13ae80b5543d82b81cd6123c75fb10f302db0cd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363659.21/warc/c730882d9a9d9ae7a5a960cdef123b5205828dae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363659.21/warc/d290f901f4f7bcb7d95779dcfc328abcb3ddf7ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363659.21/warc/df2d5fb68a81cfd2aa4b587c789fcf8495c24c57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363659.21/warc/ff7ab91a6a729490ec4a03210b1fe10d8ddd30c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/17522577d7e9fadc3739c20195636421742cab97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/2e8f0dcf5e6171459261217453aa9b877f7a43f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/619293327bd580ce5e4fae2730fce37dfc7efdc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/7e2d86c3c087d989c312f8f758e8b07368765b28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/913233664988f7145c5e44588e0b7a724cb9ac58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/a793f937e12f2119b1c010534d58f3a8ad94c423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363791.16/warc/261f829dcaeb7f1a67124c5cfc43d30c19e0b548.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363791.16/warc/6c389ad250366c910a788573a8a77026945cf04b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363791.16/warc/96de89b6ccfc9a9548fba2e0b913d5e305dba2f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363791.16/warc/ae8a82666fb3605bb5d91dc338e75329445f7a48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363791.16/warc/e97b61e2c1b2ba344023830c03fc95e445538f55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/0436d3b0c35ba71ea64d921e47b07f67815a5880.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/1e98c834daaf281f4f701302f1887d8f7b8356ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/5689345bafcbc841ef9ee3a10a171e1c4fbb56b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/bde46f995f362ec9e5b0cbb590d54adb1153b816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/cb644b9d8e9a5613f242888cdd80f65492f54c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/fff634abdc0ada3f62fee7760af3eb0235c8cafb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1651df17dd56021efec87ef7f3127bb1f01a2be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/19a1f83cb8a10f7da5d9d1d2806530895f8abce0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/23b4e2220ea82ecb4d1ed2842b312603fbc212ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/2a1ae962c47a7e14bd9d5bd39605b3e150400a74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/2f33ff3a02269ded1f858bf9b241a2a240cee790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/311140877973f1d67ad61201db98158565364b17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/33f0f08dd184da76402d2b1fc223b48bff67804a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/3425e892ddb4c766ea9df19ce823f9fde5f90c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/376978d71976fb248d430293038f3d375a84041c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/3b30e330efeefd81975a4381a274d28d5af49973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/3ca8e1ca53a16d11fa5518cd12aaaeab3929d1a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/446c7746379eea7709202494f283dd79d7b20f1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/45487e787683e03449ffddb04743f8bcf2d836f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/486c707addeacd2d09ddf42513c757f4dc1ef3fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/4944bc248f1ba7fe4cc7fd92f816466c6c7df7f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/5be90052b347b2e7c0b83fe0d4fe3c2c45ece202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/5c4a947549968580451e6d3a510547ab6c7230f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/5e588e90f133cf1f8c6c6f8d115b69c6192397f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/631987c9acfebb51d92777de12ee7b4b088a4717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/6636644dc4a49f4da1b8647b1cff26e2c9fc203b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/67de22a89c577596022ee0d6ff75820b2fd55dbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/6ad3df20529337e722d453cdfccb3b1400e26426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/6b2b9e765441a2cb182ec7b8169282c8af0832d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/6b7032aa1a06b3b504acf9f1e63ad7c8348963b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/7035c0a6802a8e698f4f3334829018b56a5cf9bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/7197c5274eb446432345f8e421aea76f499b54ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/7943225d2bc314ef74cec7ac083901c3a88d187f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/7d494a2e119e1cc705bdc186f4e4e71cb6e73bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/7f00b44a72c9875adc4041a829ee9b1d9ed9d927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/83327e426f4c7317c4086a65835525a5257f3bd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/893815d0afc255567c137ee6c55f959cf68d4d92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/8bcb1b5995aec68768eb0538e83faa4f0929a0fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/8de769399e650e85137cefc3d8a1b82895df76ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/8e8f154efae8a8fd31a7ce6b96c089aeced517dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/8fbe74ffdb7acbab6bfd0acc2f243a9b698bbd55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/9078f556b15323ef5e36dbf56d04c5bb1589eeb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/91b21958d0e53082b969b915965565f819e11a08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/9aa9fea718624a5647ff520ea826c9ebd07ffc21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/9b7f241d893f1e2d17cde6dd4e89114a29b3c141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/9c46f947e9f5bd577d4d88cbe050e4063682aa8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/9ed05d1447b2dd6cacec9a598238494ca6cc6d63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/a5e2147a5589e342b4f27eb44fd2c265264eef10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/a82d93a06940d17e435c1e1155c2cfd473c5e475.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/b2c33434b389ed45037978409532dc45109ef810.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/b3bb64ef837fbab3f471f15c67fb070756dcc852.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/b3c2b06be0e594ad7e41dffe27d02249158800ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/b6af041169ba0542af96b72e5085b02f272e4e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/b7020ae1507e9c9b5aa0c726bbe7d9ffb49f60c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/bfab6150dbb666412ec76a66d7d64b4b4383ae74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/c04f9fdcdd334c8bb4544dfdd0788c8e79b2b3d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/c089dd87e80455447f4dba6453b0661b259c68ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/c327e671cd18b1c0e1fc62d9821569839a8ba3d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/c3fc884415c73803b238acfab82b24f8112c25ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/c96b3b547a38245704cbaf2139916791ec33da5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/d471570fc99008c863aef5c17325e76a121f1dad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/d519a89b2dff3511b7edfa0392a2e417b94515db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/d8195724b4cc428fed8316608f38a3f8aee8676a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/da1d2ff96159dc186da7d05289402a83fb68774f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/dc926d90b0f029d89caa0e816de4b6bd101f673a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/dd3f0b2bdf929254704dbde3185b82c9ba409778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/de09034e68f259a12c676ae7ae96d27d4be0e7bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/dfe9307dcf261a0c93a67149d294706977032b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/e5a3315d90ae24b49b3cf46e48a314b0c3f4501f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/e6a2b898a362a9f809953a25406f2e9c39b19cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/e7945381481b296feccf5a50d429ed45d8b9840c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/e934410e663e95e25626d3159e07781893231dad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/eb07d3dfb4d53c849bffa86606cde303bf6877a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/f18a05a0bcba0d969b64c290af93d763fd6bb005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/fac38ef1061c35818127435b1d0438fbb3635e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/fc1682a7c01caf53ed752401eed0f40c4ffa9bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/ff50586af4bec637db9f034a7bf11b149916f4de.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2021-49/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2021-49/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2021-49/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2022-05.yaml b/configs/cccc/dedupe-url/CC-MAIN-2022-05.yaml new file mode 100644 index 00000000..9fe5ea0e --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2022-05.yaml @@ -0,0 +1,741 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/01957e4543d8e4f1de1cad5f38b18c20647ec4aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/0d0004189bcbfd5a1a13c5198534888b3fc437ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/0d214f3360cf023cf45c7d470330de5f63207904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/152856782c4c6108188a75daae9f3e02cf7e6ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/04b5157e8caa4e096ce712d76c1092adb159f387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/04bcb6283703b292bf0a78f43269d371d87cb8cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/48bb52a3fe0c031052b50ce0ccb0eea3edbd3943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/61f4273e6b18e37df92627e835224dcdcf6e5323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/ac57ae53ac335d5426ce1f4f9a575be37fdc04d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/b0b44719a0d77bad407ebf411f319216748c3526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/f4ffb14342497d2f64a53e48db14705fe0be1858.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/221ec79a24d049d9af48935e56a8c8aec08aa8ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/91bddf65e23d035c048b129570c2041b50750a74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/d90e4cd2a09ade032fa37aa73abbdf0fed8dc73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/e379301c3282765950dccaaae6ebed262bba9e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/ecb19008fd10ec0815fbae79de3cc3767710954b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/ecd580cc1808639da1b34803e7a963334e26ae91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/1bdd05d007d8aa7a5ce8cb7b8b43e49e52d800e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/23b68216c8573e00a5a98c463b356021eadbaa3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/40a4a6fe5e09f59082e3d8be3927c77b9993b006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/51adc6e070b843367a4bb2f72e8a3189ae73c53c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/a864b36374627bf6ba5adaffb425a97b7856ab09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/d8a31bb783062c6ed18f14fe7c17b415d9ef58be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/05821c23d2ecf09f814e5688a26d2c836f245a3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/06481a74913dedd34bd44fe206de0f289e98ed67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/34d3ca6fc4327bd96536f4b43055500d08175f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/e09db6571c3c888407efe74c5500bef7a3bfc0cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/f158031706c636cfe682512d2cfe9b4c5200e344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/fc17f68a439c2d6aaa5c64ede45a91948532e735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/011475a4640df63d3e233c1fa82e185bbd5d0984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/049f80e87fa26b1b0850c23ffd292bae28348cd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/0865a51fcfc5946ad8434d5beae112551f578fd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/2f826ff6019fc7150d610a7d4d27743962e8fa41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/8b994612347b4c7dc5caa2f624db48714e1ef5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/ac0272c1c0b26caab4f512bd1583e6238c185c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/f22d11a51520e9c048d91954a9641217bb5edcdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/03531c869147c33786c2aa1c6c7b47ff9023fa9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/3e0355aa328960f7bc43736ea1c388463472e2bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/432b823bcff8681164e36ac7eef6f50f19818186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/505168545add857aa927e7e6a45c1143272cf58f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/7669cee79e1d1628db5cb22a54effda8f75fc6af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/a4b365fd32c3124234a0dadbd157fd6883b7a83d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/ecde9824afcda83ad963ef6f81c1b0da9580ca8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/0c1aa25af08fefb47caf9e528fa732dfbba8dcd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/230cda806ce6ea2afd456f871e97e2a693e4c1ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/7331c96db6c889f81a70560194f2a53566f1cee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/7403a3acbd832a3ee22ea86b3e105a79267098c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/c153848757cc23e974d9097881844545b8876bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/d01131674d43b0d462c31860273cf8bfba5c42cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/09d807dfe6cfab8beb24370cdd5cbccdebce0723.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/267e9b3025c477b8efddd392e4556e6b68f00414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/2880bdb254d4d00b84b5774eba3eac34f96f7325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/6840d7f8d63fc5a4e93159a49ad560d34dd8a074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/cdce8f2bd75291ec31a040f5f83c8cfe4b17d909.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/e4563ff1acf6a43db7851fdf071ecec88759a284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/2fb5f847367e1af5c32cc5146d22a0a7d6f7661b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/609bf8dc6e6a3b0bbb875694d43d5a781d13ac06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/7bb8c64ae7d93f9db8ed49fc4da5c673b608f70c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/99bd40d0bbf3ccf50418bb92a22e8c7d63863b00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/bdb508900a525945e83deb16f982fea5bfbd82e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/d85df9dff177c75bd9d33daf634de43a47fa5405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/1736e6c60a6eb7ef78697f26feadae3ee261b6d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/1d0d9882dbf43a3c58996ba4de975d2f755ff09c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/b525083e148913edba760c33321a7764fa38d401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/ba498cfd42670ca76e531e6ac245f915c4d0c20c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/cea72dcfb136a308092b1b7644c3432557696f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/d427ecb72523c1299ac27f4392e9f58e028b535a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/f242270f8d926dd29eed4a0b48e9bbd1fdca103f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/0132626e670bce70acb3623793aa8bd3dc72cee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/6736058688d3b34706420b449cc2407588c4bb1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/942ff30309370df9089ce09220cda3ba6a7d3284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/aa620fc805f1dedb76f4ea018f3eb3b2e96ee2d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/ae4312f9bd04b52ef0daf6b272044b37b7f93181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/c4b502f258d2a018660a3a1a4cbbaabb4da87841.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/d873b2145b9b3b23fd3d991fe585262225bc3594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/7b210de3c52c951c721e1b4917833c03a4426263.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/7d002d4eff3a6510167456dc75c83540370200fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/be016bc46b43c5fc600ec223285ff80f79e38f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/c5d6a8e9489549a076b7e540522e603e6d02bce9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/d610ca5d398e0f059f4878ac680a5413e724d6b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/e72372765943d6b305e982a933dc0349da1fdf1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/34f255ad0ce2303a2264e1a4af2b9ea54a7ac853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/44204284766d77f54842f27932f87db1c2319845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/71d8db6d242599def70e425681c875627655446f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/c338e539638f9d0f3b1e6c4eab4505ab9f739543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/c75d59ca40754f4a9aba7f3a473c79bf184ba857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/d76648d186c5b40e646b4b1776eda90146279733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/15977fce482be2e9720dba56f10a0c5e7e533bed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/1d449ac0947c029574816daf99ae0c64336eff62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/378ab5724533d568c8486fc43f3bbaf5043e32e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/4949abe96e3aed256a541c91e1efad0eb7ae9e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/7f6c9f88c4dca88931b8b79212751d13d783e7d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/bc3d88fe4d55b186bf4e1517df8a51ed0b0faed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/3e752026dad6fe5f504e44183831eac39813264c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/6406eec1afef8751ff72e13bdfc4a47e5c36234f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/7e47db913de82fb4f765f3b835ed4d00d33b77e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/e8556dc0a39dcf2573a0b646768c0aec96c56636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/ea18d3898491f8803b06126d3e8d429185481ada.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/f0da4279c8220fd3d7df99f2d7a3f7596206a6ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/f72f02143ab5a38de9ef13571d9bc7e2938dee3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/24bb410a89b3bd73d6d5271e0c7c2b9dfecb7598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/2ae58bf4e997d1e1a648aa423811c83e46130d87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/619d42c9a627be059850bd446178b719d9d0719c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/71ae48fe998d98de8cb43dff5a84b2b8efa1756b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/bf5ba81f757deb794b469904de904087fef8b6da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/c3e4dbb1efaf09d47a45c758e86ca534c7df199d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/f268e5b48f5e268cc8f3e98ea96f68bf7a0dad15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/06042f0206d286e1e7db86641a57aab57a9cf258.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/1aa74cde34ac173b89fe6fb7d4dc01d4d4b95e69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/45908bc78e54d1d7c2449272c784c394048576cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/8796a87649e825534aaced86a16ddd0ed1ed2eae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/d3bfd13966b85eba6a4e688952ca05a909b89042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/f8cd4205ba2f52590c445ae8133239c1fe293cfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/514798f5030f3417dcc0e6faf714d428b21bd676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/5a48497d150906216e7834ce2240395fac89c18b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/6e00f0c65b7c5c2c3285685eeccb23128f874010.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/72c6b9d84d973542ae364489e27f8a3a83fe49b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/cf87bc2c6532d70201284e591d6ee51d128d6156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/edf6a0e859b7897d16acbb25ced6fcb95c64fc30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/067a728ae1ca13ce10da6407da8bfb4662dbea74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/11cc4f9fee7c718e03b52739c45b191b47bf4ce0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/4e9581a4fe1c053288bcfdad8b2d4ac43e062bc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/6ab6bf91b6d8833067a6899d2a82c70d38e4bfb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/e1e69e2804cddccb0e906cfce6193590da317c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/e91b1e0fb764b733ace303a3bd25d0f5aa53d825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/2632db44a7875fca7930b57870f82d4753d3ca89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/2757965452ab362b2dfb7b17c9c4d159f56d5f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/896dc89534fa4a350a5260798427a673a1bbc6cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/b86b214571e129d3f83aafce1693495df41ed674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/bee31efa96a1f1acdc6625ae2285e6cd69644fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/e6ad6442f600a3bda2e5c042e2bb457f13a38b53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/ee44bbb44ebd2d0e2edfe31f1fd4b927ad989424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/1487fb248ef751358e5c1a9aaeb2ef53b10bae4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/212c48d4e573624c28e98a9a396f4ebbe2b2841a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/2495e01e04ef5dbf06537db4a2ea0c9d580ab7fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/a8a24e231cfcece43c8d81696149082cc2a18d79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/b774cc9b89424b5110efab811bf687c1c56babe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/c5b3ab9eb77062d65f0ee11d5fbddf7bdf53ff7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/f779a28f2118c78f7e510c6e8ceee107262ff227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/3d119aca973e46d85192bf7d0b9ef2e26b6d214b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/92d3fadfa3b3c41469f1d444ad1069967a5f0e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/9727f245f38778cfcf9cdb1ffb795bc346f1e1f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/b2a73df0369029b366ee02850a36ff8e42d8d68b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/c2d0be009db854826efe819fa7e81a6f71808a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/c59a7e8ced0ec9eaea881c41ec6d3bc936416a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/4a23abf0e7659228056579ffacf26d359af7a82d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/592c8dfaa50dc70fa864dd1677e474d8dfd7daa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/764d882f7cb1fa7dae355277239690fa72ef0d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/a8b6c3b94c3ebeb3c6caf276e23094a574144cfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/da3022415d4939d0fc70e45bf8ee8c9372cfedb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/e95f0fb63685912d6585b3ccf7b45fe89889f272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/0232b6f265ee86788d90e86d608d56c78a6c99d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/3ecf8f7cc1c5342744bd38ba7f63ea66a6b3989b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/4cb3bdb5a0ea475e708ed907f67fd2ca726155b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/70375590134e16c978258f535698c58088219979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/a2ff91e0f51e5979d0983682d9275ba128d496f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/c34a0f375b5b9a394ec48027dc744b7a55f1f5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/1a20dbfec12e23b6f9b994223927a6b6bd7f0915.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/35c7cdf033500a18cdd4c4cdb6487fd7777a8ea1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/6f029ea6aa216860b1010c85a853d9a8d644de75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/8e3ed56c2f195c7484d68d39bf5ddf3b9e49434f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/a9d4a97f166f9e9ef3849480888fc299305c9720.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/aee085ccf97167e5cb8ddf9250bad4481d1d4781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/e2f4020fea405bff40c8766be4f0f3861164cdb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/39a4e0c4d07e5a8a3d7657c5202dbb2563ce9a0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/3aace96c8bc75e47adc279824a17cec71a3fc82f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/475452cd72f5f08e7f03a06fc0c37ba8bf855b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/51fd55ce1365df73410af73eededee0babe58601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/67a80da524a702b4ac2288fe224d5e609191c48d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/a13ed082d59c434040899e7a32ea7aae8285294d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/d13b82ed6bb82e9b273bdcfcd9058766af8a46e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/035644879682eb21eb0b4c7262cc451b6c6c75ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/2ea84116cbc5c63afd5ea6e36a0b338dc3de227f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/320bda23ba4cadee398ba135198be7fb8a2c408c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/60c1ce9b16b8c90d5548e5385813108b3bc16b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/97ed961237db1902ce43b292e89f3a200cf60d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/f5409ff88ec68ecf589c5fb5459ded48b8d805a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/2b11e6619eebe8a97fcc35a77347565305485dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/4b08f698cf877fa25d44ba75adc5b20375c60dbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/4c6337add5d5f0609ad80ea73c4fb5331c7d19d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/9336bdcb0255803754f808e876e5b32ac11a1f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/b22650e8f6e3af771aab6bfaf070b55a5aa921d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/f140632999b9f0c3981fd4beefd555a2b94a5da9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/17d0800ca2115ec48f7745f71841c9573322d8d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/1927b4c24da31917c55c432af439a6b624e51208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/2d5363a6026ee035729e7a02bb8566d280771fbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/426684e9532a22ebbf2893c1097df2dae8f22f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/595c3ff9fe40d8d0d7f2644e321d8e8d71707b88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/dfe92ea280752773277ccff9b9eb4707cf2fc8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/103ce050d9c9f1c238d416f16c1be3a46b0d9b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/1f3b72bc40cf1d39af936dfbac9bb8bb9f427857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/5dc2a8f76ddc1fc9262196bdd56914314041a113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/6981206869cad4f95da729c149f7f1d2b8ec6cd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/a382cd24afcdb4707e46e79ebe4f5721189ce226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/d41f4ce35ac7d8296e6388b144a14731aa5077bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/f1ccd92837a369c6a230cac3c69bc6b7872184a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/32937a620718dfe0b18746a9599a3f068ea979c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/5744146f3c08dee67e7d0db9dc15e071ded7baba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/590a92f24970fff67f21dfe2f4152fa404e390c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/6abb70d23955948f948168e6045475aed3fd444f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/b48200a52847cb1fa222e5f9867d5333febb0520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/bccf33312efb30150625be15d8468ecbdf26c6cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/cdab0074c91d5ec738022acbf69b26a4d39f3c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/633cff19c94d1cf4cbabe726fbc21e7de273eef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/9ecb3348911d0ad7bb4a32dbdcfdff883f58bc8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/a4ddc7c74c3c315acf87bfebd3b437362e8f1aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/b7dd3dd4f3b613ed1570f260f26c73e1d4aba4ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/b91da235ee3853008e9c4a37a4309e840a4cb07e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/debee804e159e1d5c049393447f7c93dd4b4caf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/0d7fde63d558296a47f1ff218f491def7a96e125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/1d83eb060d113fc47a563b2306a0a0e354152a1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/3f62009bb326d3f1b93608352daf608b37b4d625.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/47242c968fcf2ae0a42a92c031f29b7b08a26a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/537614ffc23ac8ec4cc0085e67a51b0fa7c50902.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/78dd0896cf490e4cd51b2da32f833bb85dfd0e38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/0f04b283ec1da803140c6ad4be96f09e0e33429b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/1a137f68d5224c91573ad4fcf9694eed1eabb6bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/7e17604be89ba508689decda0c097e1e42a6ebf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/b51ebef2afe30f934668a07052d15b702e8c6df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/b775bf9ee80abefc962b370027626217f94c600b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/dca335f1b6df9f588c9bc5cafe9aa2ca0ac356b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/25aa9177c0bc3d7e5bb83543b667c665cd8d184b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/320edd91edcc1a88a71f56db74f9a1f8a9415de7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/9ba66e22e05512b7264007541205e6a756a8e2c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/a43d599d4a8c8fe09f6b9977b3f115f34f843115.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/e1aab43c835dae1c83492da5a3c2e04bae0c1805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/ea9331745750ba8ea988a59b111073e6e3c2a10d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/fdb3843f8eadbf16452e9ec79e901a21dfd695b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/550433f89538e1ce2ee5e4716fc59765927efd65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/69ae033081d3de450530ba9ebb9d5836130f8dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/9a971494211815bdfd1131aa27eec9c1fea2f87a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/cad2df66eaccf395f8cd69d62406fbe38f5b1283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/cea44304530a155f5a8784cdae6bc082ee36b4bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/f116638b6faf8fd3f8bf397fb29551d5e8a47999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/f3446106da42537a87c03ead0ab4e79bc47e125f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/47f9667e28a8d6a888165fd08809f362146e0829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/4e797356c3340dcffebca6a79df819099048dfc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/7318c2c4737d06f12f632063a44cc317ace9c7de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/7f8a5204892d3e9b0b1a98dadc4ae405c206e785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/890995405a833dad4e39d42ebbd6bb1533879682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/d9aa4ece06eb96e1b618bcc8af12fe6ae4a3139a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/4be6ab76f106fb0df8a43c6b36d02a2e083cf9fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/73c40083aeb89ad07c5c543928d17b3bf7c50106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/a469282f111f7cb5f32fb36741d2ec544e7cd6d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/a99786af502243ddf04d0db4fec2b47023d1b990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/b68fdc6cfdbe6476510187e3c8a8a0a69877e409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/ee3ceed1235b2895c23353dc81dc49156a64575d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/2a1cd59a5777aab0e32990bc903ca24dc25760c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/6db2db9e778e3eb8c72d0eb60a3b89a949bf97b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/8036906d8ee5f1ff80df35b86a0ebd947425f3f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/9c5b315f9e1faf73cdb3b68601e6fd7257db139a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/d8d2968e76f7afbdf70d5274ba6105468e08828d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/eda6471cf9e4e43beb9191acfee01de3333ee509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/0e3b28496dcbc0937b18dc86a14cf9384bfd7122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/0ec9a56fa7fa50d938b41fdf1ccb8a42e37e6b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/44871ac73bde2fa26aad6f34a404a036e14086e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/47a51838730b57c05dff0d1c9ef2707164968853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/a5cf91cb4a425772ea837c24bf4cca308a9ce4c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/d736c72a904db1685492dba06cd20e7fd4b5f52f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/f0538c48c880c0e3d1d89f316dac29c60d19e704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/25695e2630f859d65ecc583221ea8b29a175e286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/453c0537f73a2d7fab2b11b6bb4d9fd2cbca7c24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/4616977a0ead43cf893d672f2baad6f5ecf3f5a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/61de1048a889afce77727857c5ed441ab98d0aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/a15b04ea36484cf6cdb51774f71a1b3c125605fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/c8df4236594d2b20b9eb231a23920e7b1e55fcf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/f42b3bd22c9fe0af76c5776de65105748717eb10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/2baa124c07c5ba05efd82777c57d4246eaf95040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/365da35f9456a4ebe065d177b6fff1e027bb4628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/6c1ffb9b04362c799339ed476813f348499851fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/aaee7237309fbfb51e2739168d38348c3923da35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/e90f072be255e07c66150e212ac5684a3894aa94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/f717544ff49347f56331ca78d87dda91bcda904b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/08e78cacce994146b10a779bcf53455268313232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/09f038fb91b813c9af2d96e55b982cbf9b85c9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/65cd1244d36b932334519901bb069788ff70610f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/a0325cd4bb1f35fb141c3eb89ec3ed79d57ef7d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/b91d623666e85acba4e37e6bccfc3408efdeb5c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/e02a9c24025dd32c7bc0d0a3288fe14a5569b3b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/3d7897cbf0d878b93b32a5d622158ec7f38f6723.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/5a8ea91628c5d13aa6ded705a455ffb7f9933322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/5d3792c54f5f9a3da2de9abb5b86415630bee862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/5de4aea0798bd43c36181f46dc21e40822dffaef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/90a832cac061f62ad1491733cd5152294f4c1f5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/d0bf57cf5ac3e621a8336f5cb64ee2a1f4841e7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/0ff2584001fedde6dc33b4bc502b84702cd20369.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/5b4705337bd34c02728adfdc6bf4e64f4937c51b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/7ed955c06779901b9554db1c9aa9a36684936848.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/987585729d1eca9e25df6c3a112cabf94f8017dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/ab37db57a24af41c66c96fe4a3bb42746e2a7684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/d7afd2668f1ed763130dac191f7b78096747383c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/efa80466b279d07ee24c9a00dfc865e05a84114d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/11523c54dfc3d1466ca5d355d464fe3534d4a6c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/227348714fc829719b60a391932e35c7a68281d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/294ec8a67bd58d9eb92dd633d34c15062b1c211f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/6fc53db42a1da98a9c17fae82bf62dbc3c038510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/ae7ce870fcdbff249e1b17f9a46739475db6b8c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/de347aa99fcd26a3aebcd3e568d1644b6964b55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/f98a6007576b66f348ebd7e5928e4bc6523d96a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/6a9d847a09a9da131dcfcaf4c3d35fc97c74bfe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/89724fb192c91e49b816d1559217b4cfdb876fab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/aac7faaf5947238fc9e98729ab3d91ae99a4cf1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/d70655533d807c38b102ee1f89e58d4a3fd8063c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/df2179cb56a539bfa263b0af0b2d7ba4fd58371c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/ef05eced6d39b9c94edfa2344ade2415991c4a91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/23f03f85621b73101ca2cf738302b62573ad1711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/2ccbfaff739a6e5e017f7ed6ae447ec0b11cd2de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/6a1f2ef189da188888c7e61a07b64f49d4e8fe95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/7083de2315131a2862bce94f0c1c4b6df2ed6ad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/73cd3e5e4448cc15ae2c6ed3e0772569471b38e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/edad5caf860ac3dce4d43ad3eb1acb78d7439112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/1e1afd55e9fba7976654bceadbb416f21e28107c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/95ac2fd92029844a62181a9c74afb19bb1e1b24f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/988d20fa68d0a95c15f0a75adc6bce734afabb07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/bef26af23d73dab8d528242afdd965f74cd9714a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/f9cabd24df864fb2a6cfc96bb1475ffde6e6d750.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/fc637c655ad889c535b0ae2ba8e441e70064bfba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/170b0b806a939fc6da97f3b2a44aab6e437a734c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/1d1083b1856b9e46f3704536a140715fc66c0fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/525722746a7eb0ffeea4e4392eda6f23f4f651bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/86acef6b74a8d6b6844b626ac3af5f1679f26f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/9629a9bee9ffa8de226367e95cb0212a1e6cfe15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/cca593e39f779efba4dd7e1ca85de8d378075da2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/df8a25a0272fc7df59ba467a4c49d6704c83b0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/09c7d1049659438d0e6125b7c7b7b9d5f6406ff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/124c271bb383ff76679352ea92730824a9b5756f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/1434ff01d70afdec4eb976c9dea7249b76c9f794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/5ba665aa8ccfeda97d545da36c1988a2d8151fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/74518bbf9928d710e367c7ffa075f5ecc410e5f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/92702ca79e6ce91721434d3221e30300cad43f3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/b84984a21b60563260f418ee35013665fe167d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/23e4369774d643665fadff4052f0fc04bf31a592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/2ddc1d39da3082df28cdcb6cbaea7129f56f6dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/75b81aa07f4161a6eb99d6ecbc73f67e8215940a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/88158130691b56c9395280c5e75de319b43ccf7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/9f562995311f9684b4cdf0df835c2cc89e21772d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/bc8e224b822c99e9c5ff2b2452c9d4fef238289e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/0aad7974e23c4ea2b19ccf592b9a8064050652a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/48fb613dc76234c9beac37c8db2dede027afc93c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/6899d4e77163791bfd745fed422ae567108a6814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/c676272e97b7852877e6a2a2d77bfd11cb153885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/cd9f6d5f73d33f83107b007db651e4da7d413be0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/fb7e4aef195ba25a52772afec4bef167a512b4fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/0bcabd4ad7c150183d85795baa3867c93bd47ae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/3f76c1ef43b32f331d998c1bec05d4d0e6a707f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/905e73f09d60124238f095aeea008c8a3b8e12ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/a141c031493830ee0897488bae7cb267b2e50c95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/be57ff77f1102c3a4bf65899693b81fd0317af2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/de454d669e7711bfbaf26d74cbc18966decd8a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/5e44743c40e8f4f11c7be37db0d9c81637e5f89d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/6591b6403130473d5f009c6ee5576326626f9421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/b956318fe4b8ae859262a3f38fa83df657794930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/c0a160a54f589f05f58c663ad20847399378b4fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/c195944606c9976ef84e65327b722a8695149e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/fc8e99e7a1d305100ac07a716154c0f4bd721182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/fd5d681980e36523465321af88506c69cec391bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/0a6503709d258a86587e7861a5bde55c59bbba24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/55c0e35fd2b6d18c3b7fb7dd6073a3819fdd6b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/6270929d0e486e9808f25c5d217d8451f2482c73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/65a79aa722207f8f06d39897af1a93c1eac938a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/68aa44a16634c4599c6a7216ce110b44aa42c760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/c91cc859bf51ec3339c09dc24db72008a8925217.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/f5b17ed44a81b2ebb8d302d6d6020e5d4d16e3e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/31caaffe3ddc9cb14da1a169236ff300026d393b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/45fd2f74215fdab96432f7419400582abdc74dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/5501fb1b6f0db6d959e289c8a345da199074d493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/a47d033bdafabed79520aade9e401f81bbfaec9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/bdeac1dd4665b7ddd6c73330bee07c04842f1961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/cc537ba8f8cef9c66abaf5d77573d7b55c27bf73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/2bebe6e6d2ead188cc0181978bc18402c2c0236b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/68e7e7c70f4f04118ae056b56d1497b705b8e8a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/c513e7bea81813e79b25f20270941ac9305ea70e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/e1cbe55f304602a4057bc08917551b519f495594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/f9ab61890ac3dc4c4d362501d7fc5a3a6cd9fc6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/fdf45011efa10553f328d5c001ccdb702ba7e22c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/155282ef89041d150497b8ef03aa0b19acd9748c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/94ed20b32e296e1047bb4a4c70bb520b181a45d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/b2f713ec8ea818e28f5244601b84de564f1f3ac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/b8643c498cc536627c02f9f4f9fdca3ffd9f9674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/c21a6dd353eb90147c572185ccddca1e81e4aa51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/fabfbcf09691053dacc95ccdd22860499108708b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/1bb1182e002415c07b1f58d397ab87ccb5321c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/38b70f2cc31bc267a446a2fece0ac7ff70c91831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/40fe75e64b1166d3e874f43c2e33b38688b72d4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/5240dc2678db219919325190d5da50079f620ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/9e3243cc694d59d7206aa5bf8674a5aa6ee2f8d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/df99ade3220aeea8b5b75895f11c74f41d5ec4d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/df9f34ab5399d0834e5e6149666f8767822a9b71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/3e7a68843ec33f59d66f9542fe27dd4534cd4dd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/57c9aa52d144870e7d535e817cb70035e3799e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/652b6582ea003762a4d24533c2291c5054cc8358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/6cdcaf100422669a8b8baeefe6f09352873a000a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/6eee95d03f7500afceb27dd0034dcde902e88098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/9b4e3cfb0b38ace2c169d3070c7e5846784a57e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/bdc6d1086269803d9f20d143eb5633e5f8b99f58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/0af01a30509ac4a502f296db1acf47bc666173c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/0c2d2f96a6780073c9ab0d02dac948fdab14696e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/0d43c8a1a1ca4f1015ff65dd8eac2496a4292274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/346bd5c0bea530d456e39ee9df6e215b40e6b5bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/ba0007a32d1279e65035c1284b4c85350bfbc90b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/f8057b02412b4646a557abad773d03d679b86560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/383360bcae04a1d924921fdab6cc6e1f07f91ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/4c4bb6ac04997802e363f8ce923056debf8c3156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/7674c0ff49effdadd548258065864e5e5f56d73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/8cba71ebc0b5350c049c6b5d711c6f8ec5e55ff2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/d1b81d02362c04cfffe26d8385555cc2fe0f6f51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/e6cabcd3b7951d7c5d0d3d9feaedd6866663cec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/502e0e37f43fe44e2e65a1d7a494197326bf2377.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/5e175a739330f07949d5e77dfdf4ceafa5701d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/b1c10dfe6ba09294fede561021ca9bf0e3cf1d18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/d9b3809c9583e156078f5eac8198c002183d1519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/ddf9fd16efc3623986b043fa68044ec922d4db80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/f0fe37e2013755d48e0dce669ea4611545961e4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/08f7f2bd0844f5af6b724cb82d8663cb973b2b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/3154c6ac8d2f249f8d6a42d51d0626bace9c64a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/3bdb551aae7cd18f8dd15baa6ff0466ae42141cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/588671b1c28432f03a95cacf0d523702d42869c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/890a939ef02bfee9d8b4eea9bc848f3fea70f3cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/8c2cfcd4703b75aff6e913b82af16469865468f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/94d018d06de11e3e15e49d428f252f34dc3068c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/165501bacf7b2efca9b26ee8d7e642c4402c703a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/41fe24496da85cb65136b7be470d97bef4031e5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/5ed1ebbc00c7eb0c412bb7e3a4f24c1cf618021e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/61942e16b4f85891b1c0556cfe9bd589d205ad36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/6e031a09bd995db66e1c4e0a01aeda1affb15d4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/cc07487f832ededf9e59eca7e04c8246d125ebe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/f4f74e15fa60f80d17c24adf190fdb0960d10a9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/197a49a82e3112d5d971ee6f476c3d8a4a346113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/30a7503b24991e2c1bb9b7b4c9cb54380683fe7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/57daf15d1bdd64b42410009f136a6e97a4555810.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/8d58c36a0433dd88ac87a30b1f7a6503f3c64c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/e1b56f4fbefdb858a5cc2357ed4bbb10eb32cd0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/ec1f082ffcaaf31b123f4322581ce385187e4287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/1e89b2264b9f4dbbc8f0e0b68d77477feaf6c0fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/7778d545e3a4014fade5eba2613599a0a9c972d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/77b0c81ae445fc322b37c92b0d8f72f1c6bbd28e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/7848831fc20a3fdcf52391e9196ab28f5ba2fed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/8e0583361f8a3decb3dafe95022d0cdf4bd35f04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/f06ec572af4a5b11136271f4a2b36c11ade2707f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/0809a0c5cee89a0cc0fd0f503acf85d6dc88874a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/243f595d5712786460ff8d32eee27a515212ebeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/9e807636edd0574e2036e366ec4697b3262259a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/cddfb9faf3f246e9f9d33fc4b656407b0f192618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/e985c424a77ac365dbae51e77ab99765a1edb290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/fef3508d0cebf153e33d51f22d4d84961e48f8b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/64447cf20156b9bd46b7d6d0b51953ea8a297e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/74aba389207ad35b1cdbc503848037c7635c577b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/a67e13a0205e8f98536cdf7643399b0b96d484ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/a8e60f8988bbb873632808f611c67847e4f8cf0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/bceb23f83a7f9cd220019e1513d096350b473892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/c6a22eefe13cf1b81b956aee2083f95b0ddab949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/e9506f823a73e33352f44bef109b8a20fd78ba67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/1558a0df9eaca9624d332dd3e4cd115ce6bc8986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/275f1eaca6818d086dc54ee540ff0bd385fc8d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/313c4ce0f13c15ffe220d8488cbed10fa8338e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/39754442eebfcd13dd31a3f8a3d63079d51d4db2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/7b78c22e103c9dc3bfff678ac6a8bcbe4c5cc104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/9bf0b7f937d8475dccd65f80206b4c072a2a2f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/df659a27bba89b00019e21d7d13fd88488f585ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/17a0e663d398c6559127908ee77308c5c96f725a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/6815f020060172cf6234d639a0d1d8abcfd7c9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/737fabc3af0aeebcb640d2af265ad1a8cf62789a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/7ba721228c99f9541ff3a7aab1e4e2750f36d9dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/837b9fc74795d36dabadf682eedb7e58724a8569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/e2e284f3b6cddf338fba6d0abdcd3bf108c61777.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/0412f0008ca429d6eab362d3b31323c6bdea3091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/1146f5305eadca3f4630cf298e6e4c70adcef845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/215dddaff1eecfeb0a8b96f0559b6e4c55353430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/687a6ca38e25f1f2e3890ee939542695faad6ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/c4feae52c50b608b462033c24a601b3939c6c56c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/e002738488673d3bc27ae1183c8c468a38b3d7dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/017c7962ceec45e3f477e2c29e7e07e9779e657a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/04cc0f4264edaf09d4b22bcc5d7c1b9490c35842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/985b488ef7653552c9274f528b13e2e35b16be4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/b22bb6430a2a9699db0df1ca43fc2380316818a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/b6427bcf9621ce3ca1f0d4eb65e7666d43e98d59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/dd79dfadae69bebd1582c34fc439a48175a4f3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/07e637e9ad65f3598e1102af3a45084c29cb36c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/12f35d6b1ae580347cdadafe79e52c8957a0cb1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/170ceb2c1c8f49ea0f43e82b957f6946cd1df092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/421970829e4728b2608366452d865faa02ea3662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/ae292e85a938638ed96408f3908818177c827069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/c30422b3c31e1e7aad7020c32b537b8f0d80f553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/eb85dd7b83252ca29004dfb9487e102fd91b390b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/0eba6b3ce83289f905b7dbb0179fdfc5960bcade.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/381ea289075467f4c689a82682d209d4abd02a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/4fbb7828ce65566b3e17c90118284b039be05d71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/5043f29e5601e96f41f799f6afbedb4e1dc90c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/82b9752ec47be831ae3a4d470e7dba4737fd302d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/8a9490d3e4d839c74f29209aec0abc9ec5b20272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/f40b0bec09540f81c4fbd55b9e2240d222bcd4fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/039921206947dce5c794534dbcbb49b48b20cfbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/5fc0722fa02720ac7bc1ee342944da5a4efa7c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/618144f429fdfbcc148c11fd2a8453f92a226139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/a88e5f9cfb3f32740e28905b081430013998d6c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/b40df5d726d3e1134ac6e5c9af90331e2e8cab8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/d759b9bba30fd090f003998b16a92466f5347153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/3291555afbe5ef1ee610849066614791e7873047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/5aeaf4ef040a4499255fba784a1e1621ab0cc0ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/8cba8ce6b577778999855b62a18e133a2d6d4cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/d20556c42173f001214f84b947267288fa006a33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/fb7f141cecad4033c5b58f15e6963582c3a1a34e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/ff6fdd93bd2dc0e0a5d8e31ec27d979f9c92fafd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/1c3559b82f0c8e3e2105c5414c3baba6de56f369.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/6805b1a42b00b647d611a8f9a461134364ac119f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/a559bd06c9d650d91be51c558ced2756e2588388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/b0cf7ca0cf7125719a4eb65dc008ffc59faabf3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/b2fa38c6d1aad1aa0f1fde75a9d4fef534bf0d2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/c1b36d615307f0ecddbdce8f6720e2ebbb358afe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/3ffe5633f52bbf78169d41bbafe7bd1ee8971506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/45923bd5358d47c4df442e31a567427373b09510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/4cbb30fd9e70fc59b14914a08af0ef08f993a593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/5b97f703bdf1d2527220cc9224e9f7bc1212b0e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/5c7cfba0a8cc78fe5d6e87d03960780152edacca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/63028a38e8c74e2032e432f18613e6ad563966f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/8210ba59e6238a025ec10868fd31c9d979dac5df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/2a96d742789d827e0965f6cd9ebca7f8e7f30832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/33423779887ae9827e8512ff98638dace206967c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/4993e8de836f825cc35f063e4f8e8928aef7e1a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/82e987eb9d476e1f273cef4e6611d4b4e37c8799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/b6a06189d739fb7fbe4b579735f518528dd0da6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/dc3cc996ab11e8d1c6a5a1f81e802736ddf4f2f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/ecb0de6ad8e0f9b0b312fbfa2d891b185edb6998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/1a64fa6cb550292ace7bea44d4a3f64668f6b8c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/20d5019ad5f9fae979a32e0f73461cadc55a5c3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/3d3279857e9ac1d26b68baf74856c7f2847a1ac9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/751593a6866c0b952a41524929198552e9b78a1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/8d10e3c6ee8f7a5598b4f6d2a99ff5099556ad4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/90b1a9948053a60da042ade475693d2dfa10a9c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/002f62fb1c508e20a921e12c3dec256f8f8febe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/169f7a0b1db3ceb2f04bb319ba4e2c9acc96d50c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/59ab0ddf2ba2c6722e7a1999a56679c2c1f45f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/5d5451d6e0a65bf9cd09efc8d6452b6d3a31e3c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/7f21532c190e739291f73a6bfce165073706bb77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/a470b9a41337d33860c521316756d66554c8eb26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/490b9fe0c0924079688e6e7cc83650f8e5126b9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/4d1953b8030af60ec07520af93aaaf49585f523c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/5ebd8328f4608ae57bbaa55e9095df3bcd648cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/6bf5db74a0ad2d8b8497f78252e67dd117a34ddd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/784378a00806f6f72238668120fd54988d19d1f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/85ee6667423fe998a123498229ad3f4b9dc237fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/03e090e91d0cbe79fb69555219a985a1a8cbf596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/259c988f3202eab044f94d1b24e864ec6ac3903d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/37d6d99388ebb3ee786a79ed1611115a85f0f382.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/45577c45327cac9b22b181cf3d532f8d3f7ad68a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/5253ea66dcc1cb16d8d2ff5f9d61679e51211df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/be2758979ab3818c7566ab4440ee5d2a4a9f64bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/c0f0e9cb4cc81f9c9170110bdfff08e7d7c328e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/11ad2171b74a094e3fc53a60b0a8a71d25f14cdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/189a8ea66c6694c78684d2ead6cdb5c1bd64a0d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/2f6fbb7751a7129f6143305d231d3e5995de5735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/5e1e2104e779d5d52663a077e30ea6e06f2b12e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/6cc8575b3256a487c084c5cedc9bc6588db81271.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/9da70ec0159796c82ec2b84d506168d553976e26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/a92238c04b4190c41ab16ebf8881302564b1fdec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/4056fb11473dd5eda6aca84a81339a1cf413e173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/4cce47982c4a72a0bbf4bf97bdfe723734f66ff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/9b55811bf2c310e7708039e15a33748711bbd0e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/bf554a42d8d8149d5fe6b4f2d3f454ca2a8da936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/c5b9a96f7b7cff09ebe2e9635a1117e306d425b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/fa0155e122a61f0aba96e63348d2d4770566716f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/017b34e56e44ec957faeab2ab9cafb973e22b8a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/0cfe0b5428a211a56da42a4a8dd39bb2c8ef7912.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/bbf00cb5defa4721ef26072c04ee7ba1227db7ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/c77c9bbfd40e4146a673a4a8a2f75ca40c879065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/c8c851ecdb6824d4a36d6737c1927c62db48f33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/f80751a3300af4f39bb998c5d0d8e36b7a55bd1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/1e6d8e6cf3c28e8010da90aedf26d120179b6cbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/40d45cf69a16ad009156a954ca8ee048d9856107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/72792c4166e6e9cc214e552d762f7fc859612ebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/8eea3d5b485d139b24ebccec2665d1a6f3a5e7d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/91079ae4dd09bc5d3e89dd4028ccd5b80f99f399.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/f22706c89f1ce756f0378d2ee74f535237e7aa88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/2fca7571987873507e3d37e0fa6eab122f62c272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/625c3500e037f531732d66cf68a182428062ab2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/62c0b588cbc6dde1484f4c3e36f578ef1f80b423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/88660a1a7f39168e28aa6d3a5a8466964a4f4c80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/96a7bdf75332cc64ec0cbaee40cef4c1fc062086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/b432259580f86a969dd0a70537ed8a7f404fc47e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/f2e288042a5a6b0d9ceed0354e102d0e4e416c11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/36b483b0b7662a6aee2f6c82a88d019586b6c112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/3c7a12bcb05ce07b8fbd183ba1ae67995a71b50f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/40e94eaf288a16bb8e29b972f01f194a0c1568fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/50f174eecaa834b2c8ab871f0d5fb8cc6290589b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/54f12c896a887deb23dc2e65ca03864369edf340.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/708b00b17181997782a7bdb738d1af85b796b4d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/ffcf3aee766ac7d198a0e8337e37e84d7b072005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/499b7ec530a8abb36a4a79725cd010838ff2aaec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/9b080f2fa9a69700b991e548fb08beee6271aa90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/a160367d638634143bd7d1c332d1a5a034483b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/d2277f490cfa4aa7e286f7486944fd5f67b2acce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/d3cee9f4445ac70b6cabb9df042e6168e40f166f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/dcfdbbfb6e676efbe301f8294cf2e613fe6bef90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/0edb5beb7239c2b5d6c64fdf714ba65c162b9550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/1d0ef19fec93db08aaf3ff3659eaa7d4028c9244.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/2ada04b931af0831d40a143673e514a43e82c7c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/5ff9035ac68565bd84a7ad215ab803378b454f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/75dd6de3d874943bbf9f7b78fb7ab196cbab9bec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/929bd76c1e235fcaea17c52345162dc227934112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/0eea748863c7aa075aaf5335f8b1fa896f9da90d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/15e5dbf2e9a99c098cecb818bdab9b8604c2f45f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/5ad40f62f7356f3f91d007d659ab056a10dae3d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/6b895ebd9bc2e5a584a0e49446a7edfaf53d375b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/7ff6bc7597eb4c23711074e37c00e66b80a99b29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/8e9d305790a63484551c5abaacd3670480d1c093.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/1243ed5b881bfe9c5563417004f045dbe15d7d56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/8912ae720db200d526bb4aa79d6573656df32149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/98cd72cec16c29e0b4a2ca7b3b52169f1e037e0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/bda3279b1fc081fe2df88e99085135d6551439a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/c8a23ceafd8d41995565acc805529e0faf538f6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/d7c0ea6e6cb1b13d98a4be4d3a9e1c628df7fa8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/e8571585abe8f6a9bfa7e83a332b524dda94de39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/1c0d48411a43bfb0c6cea04bc852bae22e86aa85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/7d88cddda806f5a614224e189d32f3120d439f3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/959d1fa36b15fa1f86975d30646cbdba49d33bc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/ac61bc8046abb6970719641799a8e5d1d90db435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/ad3bf0eeb8d9d5a9dbf7519fa6f51e94e193bbf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/aef6588972841562f313b29c6cf2cc73881d1281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/b4d822080f5662a90937b1110bb5193d0d50a21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/024873e8ba705bd18c068fe5e5102a6455a2a475.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/54d41ee02703e60a04b6e50b034dc35023b29f11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/5d2335ead8142d94dfe3affe0350ca7941aed7c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/6a54ebb9c5174a1454402a14d9b1c024de1f4363.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/9ceffab68fe6b9ff828ca718916277e8c7de1582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/fafb0aac8ba519c834eea4b114d5b126db5f1c6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/1f9c94738b2021726d6a8cb04b45abf63b957cf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/4b32841ed1572b778d3ff69d1510def9fd9af27f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/a1f33e05da0ac9a75c4e4202e8ab7e9e4a5eaa4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/ae5e86243249a945b6b97e9361cb381ed6745b05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/ca9f8e84895021502a966e159525d9aa26f2b65b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/fd0ca87536b2ce0b8f641e1d3cb19ba3024f7dae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/3b6e67d0fce395d38c3219238f6fa6bc72a84df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/69b1ec890bf27db4b1b1f8e765f7fd4e3b704e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/74d63730f2242f12623fd32dd399bc22beccbd38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/86b10a3f2adfe301307d00f1aa59b4a656be4343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/aa2cae6eaa5e703f287bacd782af9f8273ea7447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/ec2df7605554b28c5b2104695a073c79185cc326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/0cada891111be6c7f56d860240795b4d7aafd460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/448ac08e52283b7ee6896a8051b4071eca19daa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/5b0f1d09f2f7400aae2f4637f191edfb56cd9082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/9a05696958fd2fc898536179a3eb9058e150901d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/cf255c90b0570391f0138889db88860012392d77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/fa9073918c227107f5ec245c1e50da636a19fdda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/faf1255c8dae214b94255ff46f96dca72304b66a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/18be7e876ffb96b616ac7004b755c26699ece8ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1e1b3257d91799e19871bbec205f88d9fc5c77c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1f21a86060d48065d8103cc2857bf06ad9052a08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/20052cf3dc260cc21552c2ff695ca746546b48f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/204abfeb56826eb1359c882cc3258b00e62894db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/2372b4a951a2de1bf03d61e4a6eca00327dcdee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/2a53f1415b4f3f9895f8e4b65c8b5baa8d4bae7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/2ea8cf2a593786bef97a984d0ad7003a0c20b78a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/3642c7d72c9c33c8c106903ebeb22457cc0b5d63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/3df5ccd1df8c80cf41015e0639df670f18fa2b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/40b77bf2636ed78f3958779beaa17c3a4f655b51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/46d9aedcc9c090f64580862a540ee8f8727f17dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/48cc8dc1cfb19bb7a0e3f7f5a8c3ecef3090f33e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/49ed749e4c5d1064896a2a923e8e68c1ef38abb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/4c948614cbab2e2ce60417a7e8d197e815d9b442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/4d34099156f9b2d6ece33ad45f92ebff8dc1b2e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5127b5ef477db22b1351d3dc25f57faf6e43eae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/54282e50297b7f2a409d0d43c379ab4f3fc2c17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/54b32e6d084bf8fe4e61292aaa114c369d094889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/578c31c748901d70b040632d42c51071d2115699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/599a8e37a4bc1e6db093b1b687c08b9af352fb2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5b8cacad26b4ac8c5aeaf6782dfe163419d491a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5c422e8972f5201001c4d48d93a2bfdbe8ee562e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5c5bd432a6d7ab617939f6e04bead13b2f55d33c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5cd372087fb660591488b4accb422372aad2abda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5dec7d5591155094f58e5512431b097276dfd90c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5e3c2b4f6f6577a275d1ba335f6bac5951676273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5ec741ccb0dec5df07fb3ef36ace076b89a751ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5fcf6a271b0f70164c1ea6f78463206c33f8021e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/67910b25850c22de7b02fa926d9b42350c71dd73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/6a9c991bc56d9997784b6d95036dd46ec53a7b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/6b52d20f600707d9e5a302d364aa04f54724efbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/71e71b61ff54941be189f947ea25ea8a75bd412a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/76b265509dcc83de2f48a84aab58d1fc3dffce60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/77ba3b1d2ae3468f54c174387f86e5a43ba84b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/7d76933badeaae4dc7f16dcf9d41491dbd7c74d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/7e23fcfa29afb56e28e0ffa2df573648ebf15d2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/7eb124c4dcb418cca39012f598e0badedda3c560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/81ea646a40f630f07e3b4a5967a6d5a62db25e39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/8264dcafe8579f1269b4b000e14dca861792dc63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/842ffcbfca68527793c16c00a4563fd6c3f73b11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/862861794b967df82b623e4633f8cb23384b227d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/8974125b81a2df7a30afc89aeed1451959a74385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/8b6ae69b51e0baa5f791f05f6e84e56a0a54b27f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/8e2da93b3504260bbdbf34a1f64dbe4c03641457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/9005ba9b5879c170c5ca140eae8151d20ed014f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/926dc7f87f7657837557afbab9d6b30c84e678ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/93778a91a3b1c8b4aa51b89e385234c2da30f7c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/9858b22bcff6d3a3c9bcc2e9db0f7a8bf8cbfba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/9d73a92d3429f5a1c4ea2e60c231bb7d1903e823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/9f1e56fa27a974df0f53995ebc1e48d8c3fead0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/a17122cc8273d0a0df454114b120d77e3bff5b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/a338c3c588d4a139a7b0409ba80226f499142c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/adc6265d6b8211397f1f8f1ac5c93fa8e18a056f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/b0fb091efc3f8b2871adced12b92c4b16077ac83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/bf00a326f88258e3b00faa2930ace11ffafe2a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/c58ef2969e496856dfef62d3c07cf51b3b2bb129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/ca39a37e29179386f8e45405663ec89494d56a9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/ca95f4ce0ff2fcd2ac46eccb293074ab6f1c8b9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/ce2690f35c02dc80fd68a00289d85f9a318aa6c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/cf27487f4325afc26ba070442fe377eebb36ef2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/cfa1ad0f8e431296d04f5a1fc5ace574b4f37aba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/d4f0eb4debf866e12f48f2ffcde799304709080c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/da0a57ae76b9786c337c9dc47f914cfed6427b92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/da8ee588007d9a8395ed19bc2eac4158c6c0607c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/e915a252d70a2da4a9c6796b95c0a5a980df80ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/f2a028fa842aba7df7d2107fa00c8bd1f12e1969.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/f492d62d25d64f2d60c0a6a19d4a8dfa64c2d225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/f56f51c67862ebb8660d118c3fa1528871cfe537.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/f9716917ab90b4202770d3efce5a38ad500b9004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/f9e05072a100dc8af2cdea98f70ff10c019170ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/fa27a789d1d864014d8df479034734a7dfdc54c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/fa8c79bf4f639fe3e9fe0fe89673214c10fc236c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/fb80d68c87dd42e99fc044dc5d59319aef4169f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/fb9ae7206161c9e0a560d53a7db93ed76b90596a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/ff22e2e4ed2b694832819422c4f20a8086363432.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2022-05/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2022-05/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2022-05/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2023-06.yaml b/configs/cccc/dedupe-url/CC-MAIN-2023-06.yaml new file mode 100644 index 00000000..d2c272df --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2023-06.yaml @@ -0,0 +1,901 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/018f55848e027ba4fa534e282ed49e0245cdd075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/03eb8d04471961319f58bbb66ab010ad040ace76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/049c1b2fbb35f711d0d5b24cabe5048d7941d060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/09d1af73aeb4017fba0b3c9135f336617043f72f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/0a7213f430ca4128a494a4cf3f927fd7c837e3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/0a99aa8a7409303fe9a0b3f0b041e948ce5d792e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/0aa8a1cb354d797c61f126a6a6aeccc88eaf6b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/0d8443926e087b47deca67587edf95cbf297bb85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/0beb6571d4494fb07d760c373fcdcc5415695168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/21bb2b9a341025c65c2b63c82d2fa975556f68d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/2209cc305e43869d2f0e85b7dddf9616d39d642f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/5977c6db8182df69b7143b0982f88a32323abdf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/7e73b140e875dafa2e803c1eb88c3397d635d2b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/82f025787ac2dc1b9f0fa2d99bc07849e606d71a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/8f76a21cab03bab748265d55055d736125c15990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/ea8db98dc558966e1601264ff55190e76d25457e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/0d8a16a3db501ae38b59c90df34623c928dd4ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/3d5cd6d251bfb52f36196ff130c758b774993eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/5e48a354adccb23f164fdc666b785ac72f220b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/5e776bafb23e02170c6ec41c1c4ef61e8b635487.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/6bfb4893fb9e895100bb0173abf7e664097bec61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/7aaffdffdefa1757ee9a1607ad9d9bb883d5e83d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/84aa7dec4b64d938ea24e1da468ec6ae8df1e36c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/ff9cde93e01cee3998fca7b38916fbe574cac5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/3214536ece5168950e192d6c99e85feb21ffb3d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/37adc19237eb9225ca75437ef2fa5b419b2726e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/4fbefc5b32c6e6de7f78ecda2ab99a6f105b651f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/54ebd9b5b774692ab4c14d99fe0e22b5cec22adf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/798db5cecc9664096e728844914bc5fca7aa33c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/d6a7685d69461d5626633772efea2c7200bb6499.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/de67e3360355c0b4408c3601530d060996a91f68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/e6e5c3471dcaf879a0a33eb1a32f251a3cf494bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/1ffaabcb6a81fe4218b0c4f38146c89b81ba9c3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/45d9ef13e8ad6db2b7f300d7201c798ca7399480.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/514cbbdefef974514a49190d1eaaf077c3f32060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/888d86fff3ad08756f36d2b99a0c4e5dfc8c46a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/acd9ceea3100cefa6df2ddc435dde57cb0bc7e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/b8ebb966d1d19b41c766b831ed94762c61dcb024.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/f0b3be1718117fe36fa99425039f74dc3fcced96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/fdddfef94d8021d1ca2faccc2c975c16f19b5e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/12957c4905ec884cc8304ee44c22ea77d6c37d22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/1ae2473cc68e717ff8cb73b6664bfd3103161224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/4a8ebbc52b13e4769af20d05598c7a4594dbaf87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/5524fe16647ad0cf69a609860a757a223efe68ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/70f8425ee1a3b968382e5f6b2debc5d3ef0fafd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/8ef3d37e359f5a39672696386fad7260ce3139f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/929718a22ed70d8ac03040b6f87fe8759c450e2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/bf6625e959b155188e8d3b2b1f5f00f31c0fe4d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/2f8412db2d65afb9b236a029be1c3dfb29528916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/463896723f198acaf8191cb96476b7e59f8d3434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/497b7b1d0bc9d45a43328a2556f4d1331a777256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/9feb70874d59576b06fc5d4caa1f689e8250c158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/c561be67abf99578e97c61bacbb29ead6286dc98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/d04f459c3b682dfdfe642b19cddbe90f9e328dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/d5c1d600427103f3132a4d957496011c39937ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/e733c9923ac67f6435a2bb6ad15afa958759b7a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/2ec08b041812bca61933f73f61ec6f3fe03e1a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/af19b97248e6cb4ab61250a51fac7eadff6e043f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/b064b7578fca8a1e4de4b88eaee2d9fe79adfbe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/b136b87684051a370ecec179c6a8bc5225ec5fc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/cdc8778fe03c4e3933ef299526ed00f512e469d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/cece3a172c7cc2e0686e175e16dc8e8e5881b4e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/ee012853c6840eeb037eb94a491928a6715bb63d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/f5b5e39fbd8d8f666380f24012072abbae3bde5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/1db847a3436eafe1141253c0aa2deb261661c606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/4d18e9c9366f7aa159ade193841c36b953b9aab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/4ddf863aee65e3ab6e0078447263a468cc0ed4a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/54a2ff2a01b3236fe9a7925f380faac25aca1ce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/addff1351114c36e782816a90766c0a309620861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/c78373f80059cc65bacef4b5691fcce3afe91cd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/dae28bd4827630bf401dab98c20b17ff5d8480b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/ef1d0c013dd5354dfb58dc38e4e8a11c2b3e24ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/2ebc85ed9db09bcd865f04fc0abec98663af53c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/375402a82638bf1f44a41fc0686dd9e36a11eb93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/39f90850b415fedebfb5fb00b10ba1ec65e3a0ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/56d27e86afdc294f980144d88f8b756f2206c792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/7f6c44e7b9652f5717caefd9154c6342fe303d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/90b7a254c59fec9e3c2455bea205e846e8c62d8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/b610c719ece2716a841b579773292afe3d819fb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/dbd1be0b2ff2fd3370ce0f52db0d1a2e3ef4c5f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/0ac35c5ba3c7c62ae75c02644ad89549ce1724cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/36d5e3adce05ef2de035cb64ab2c3fdf022a7e27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/3dc56f2ad2103b687f0862f65d95d859804246f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/54113acf016f8e0740bb2525772e1266902c10d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/58d2e9606eff5073e7f178a821ee6cd4625bcb2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/8cdb8e34793bb61556a7d8de5637ad2abc8c0502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/8dadd0fa1bf3e49b76791741a31595eeddb51ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/a3129f3748d9de8321a38a8a5affe1c73bb0dd2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/1d5462325536114c985c7cf89f2c07a0af3e16f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/2309d0363a58c5cd7eb9f6ee51a23c6474e93e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/60bdc558cb1c9e116fce0e9bebfe6c5f0583cb29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/7a3153e634a82744c96ec97f4fde4a281c2f8b2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/e8e6e749bef71560a4f9f1178f2b6e025ef5a2a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/e91d20b599cf31d9dcb522b77466b367a5741cdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/f48bf239faca6d98e52df523baf45473f484aca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/f736b9ee2adccae5e8e1e236722810d829dc32c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/2646103e25c246fa289a3e1445c7ecfba774e19d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/28062e4103b83c0a2bc154770386571ea1e89ae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/6156d315f951f8c4e70097cd299f4c8b50780471.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/7a967ce0a6b75849592c2426f2de6370c5b7294e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/e1b0ecb2a1d6405a3528097ff37d6b9a561fa9a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/e60e3120d9898413a039554ec8ca34af065a995d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/e8d27cc303bc2386f26a588b94f5abc035f2ff1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/fa95afecfdfa9b5ce0fbafb3c8fbd47e517b4345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/02586e168feadc6d440c49ce8e82d9b90beb3757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/09474b1d2f23511a37bf1a37b27dd203f36059ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/125d99cda68a7aa91912f9ed53be9a15fff83a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/4a2a8919b1d76e3bb5321e212c295fd6eb5e179e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/7ad79929eabb434c7cb73411ca7eb247552ddcf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/82c630b8b0ad6204895c7701ddc86bb23fa2c618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/97d418087f3c94bd97d5c03a96659d8d39f837ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/f551201e65b679fdc7054d00313e08dfc7f452a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/01c018b90df50fb3053d2c2a59da83d7c946d30e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/1245034123113d3d0350892060bf0e87e24df0d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/1b56bc0e43f3280b4a0f3f9a72ba592832a76352.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/c6859c15a4b220fa818ee3971545893ee8c65617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/ce8b61b3ef1c1f1fb6387816ee1abe752d2be6f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/d836ae39715792c6b8237d84cb3e272176632d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/e15584682626126e94119567d799f03c0b9ff73f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/f183ab5108054de1697ab07f648ee35d0bb07984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/103a1e3dea81ee949e9c811bd1c54bf8462cda92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/2147408fd3d08b5bf07daecac05969943ad89624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/68786d919b5c515a049cecbc7d5dc79ab43eaff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/6df4858a7105b197263fb38bbacb76e1df13cb67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/8eb6d5cd1f58a43b62fd8ba63a262dc79732c8c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/91e6fceb2a81932ed53dbe94df1e36b201f2def5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/d3159a2fffacfbd5209a6376849297b22584901e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/d71324573ef3fa464bcdf43522de81fdfbdeeb2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/194ed088c20c0738b9553d10f7f1d8898906b4b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/4f61cd050895ab293b8d67a152d2ef712ba0e806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/7ec4dea52efa34495f1dbd1724e93ee13067552a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/aa003cd0e8fb367fd9e1845660c2c80fd4b5a336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/ae0b7a23bacbd9337b988d4aa1ca93bc0c9e82ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/b6f3ae75c609dd494dc6be74e57ee394cd07fbd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/d74a0676564e7dafe402e3c98fbaacb592383a2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/dc9e7da8de20ec848df845545f973c6326a2627b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/00da57217f609a641c03f44a1d09536fdbf7ea24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/1a723978a7327a15fb41bc463c654422707131ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/4677b2cdb38f82010b82f6acf15a8df54b9cccee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/5cdd6bcb7a2c5827624c128af4d3ba076cac3c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/9f59f0e94a0a8760fe0f8c05c6a6c76586383e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/a58f468d62cdcff4b36594305fd6ba0580c295ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/ca8dc09da38934dc8d90179b0f021deaf25ae363.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/e861bb49583e2b5efb118431f9effdb56940e353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/10e5880d9211c24d2ccc11f68cb5611b926c372b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/1aa6bece9eb1b80106a75d4c5786a4a765140384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/735f8f9ea0a9247ac624d401be644344d7a96043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/757ba5d00df48a4088bd5882981f033ec5b61400.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/9b8fe4d9617552239ba1974f36c1730f75c13e88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/9be0e5ceb64204028c347c443b1ff1c8853f5a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/9f6c27b25c66fbb6dfa715176ba04cc81475d36f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/f8d2fb65a8fa8d3bc3429c3ab54b9da46b859ca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/09024d524f63c84f0e4cc9d6dc93b6fc8fcfb51a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/4dcafe288e632b40196dc1e6bd75a947183aac8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/5742395a012e8db13df4f8f9859411d37f8ca124.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/636373e3004ebc5d86fd18e92b08016372adf2ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/929b05b800cd5033b5588766f18bba28e165b841.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/b692b76e442e3cdb197882217ba1fca1503f01b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/b6fd698698de1fb5dd684fc31e5cce832b5c6778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/baa82d2a783925c505218b24a24c3198f34f5d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/10f771c225ecd5c80b0b197896fbd1137827fc35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/282edd453c23aa0d86eaa702dbd80a672ce39e68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/414beb116f56bfc618d1c94dace73834f1c94e08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/56884a4ed8f91319273609b96a21b9a21b705c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/906357177cba0be5c0c8d824aa905fb134e6b80e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/bd69acf347799b440db6ca5ec6f3adc84b3ed6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/ca3f354b5201d00d2b8f74aca440c87c4f01f8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/e2f2dd23371a84199f995fb47c6e1124eb4ce670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/30610d234e709edbafdb8f5596350c89e6e14c35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/38913f71ac12c460382ff613afd29e03454ba972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/498ca9ea8e7470b4b2fa82a0177a296788e5b8cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/701a674f5cf850aefd9e64dc6254d9d8dde49cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/74eecd98a91035838588904977b14022dc6993eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/93598c83f38e9a96149189ed9f660772db6b4793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/c9b81963c99127da2f5811552839ebef7c0d8ed6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/cb8486de1f044cb2f784e80a5d2c61d2c3e03675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/149cacc46cd3a6fcb8a912c9de45f26a523f80e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/163606b7cb9486e1d7556646e035b3e070f457f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/1976e0db10ef2c42fb4a025b73923d4ab079df94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/6964f5b4718babf9e29931cedb5730fe84dfaceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/8f402425273a95ea5178fadefa23ad4859f54607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/93ec80f9c8f3567abd7a758c9a411d965c676d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/b84bee2c1129dbd5f00e7c5ededd971990ffc569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/f6dc9ddef35629a771448f23bc7113c50c8b7136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/00accbdeafb8868d102dc376b7379f4f489754fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/3344b7eb102b353d2ed3926951ed3a37f46e0628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/5f453cfcdf220e15abb1c6871127a62909c0cf27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/7a15ff3fd0fe5e6a56438de5eab0760de7a25d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/7b96791ffc0a8256e9929daccfea4f6f83cdf4b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/7dd561770216a74f5d2b90502a8e4cdc2052322f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/969f1549666e823adda4e54e2302ffba0e04d631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/fce4384477726eff3f6ef2158b510427d15c7a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/6eb1ecd27c0524aa0248ecaba8f465b1e948bba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/7ab94651cc830e46350a56050ae7cd5e718585ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/8ddcb7a2795c81a470943c81eb050f9b39df4cb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/aa4910c383cf7eab7954cec970df13b5c7d49560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/af314aae9667acc1939c360a491d10a7ab9f7392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/aff4673d4ca9cb1bea2fed04464b502cd6c84824.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/bcd2e7a27a85c7f4a4ca1848f9cc8311bad04acd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/e8d504389db1bdc84a5593df9f7de2e7d4db0b60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/046daac00219b318860bd5063416875aec328d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/0a4891ea9ad661d2ac8d30afd1542989d875d983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/1c49240d8e00253292ee3c60a6e4407629798bf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/233e6d25d56135a7ad74a8005a7e00d6f7f97256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/90f6e4b65ed643f4f003221a9838c93293091014.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/c17066ab7ea1a431371765774d7951f64fec84ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/e81bde6c52f1a5eff4197345baf15bc41412eda3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/e9ff2fe22551e97695a911886c2a02043e7d7a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/1e07512803cc21e147e8de1cd605448e1504a33d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/4085a17dccdd99da71e273d08cb35562d9f72182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/4c3bd5e6492ebf6f24fa514a24f389a680c73224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/7b652a77fe4301496830c7473614855cb7f272f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/bb54059d307de888e7e3d418748e8c569e3e425f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/c364f83803978b4ebf7cc7a9e0b189d82ddfd110.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/cc10eeee46971e3633a19cbc7f60fb74111f976c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/ebeabfb7022a040d0aa357437da2f6ca434c0514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/31b9daae7bd5bcd6ae5c5173042b80520c239075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/32ee7bdc4da71f460f87ee1eeefd10e49b4b2f4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/5c03c6be6d30359adbcf119c7a0e00b2c9e33e53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/93eef1bfaf97daac149683aceed51cb643a2b5b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/9ce598264d29853a9f8b7ebfecbbb5cea0781277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/bd1001af587646e1b2f6bc7c3219190d4eb86c7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/c6bb5e998a1019995b9a4b09eb0731b69954638b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/d0876a483ca9af69f2e0d3927e8caba2b5e2990a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/0af9a8b83b1fd0869a05daea47a41a59fc6a86d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/11090ac1fd99bf768cd73ed71a8411d8da52e8cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/13003973052a7bb9241c4715bb714117ccb4bd3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/29fb6edbfb924684d0b45c7b3602110a0e5a3c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/3be9b2b397b0869c08230159aba4dee157e0263d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/53a8b3201f796229372ab6f48e1de32cdca1fbe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/e59f09d6f42792823f13b80b75e494810e698171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/e69754b0ffa628381087e7ee6c421585713253be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/0dac1953ed953499d27fe92004cbadfa2da35a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/11e40cb66eb0ace5a53fdfc010cd95ef7e397485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/392cb50f35b8973ca617da413222ac87584f60b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/474a6f4f1b8285e605a492a113d8650809eba290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/90beeec0a328e78efea61fcdafd2588989fe67d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/cd1a513082ae56408439f322169d020c06259b04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/e05a6af253f9f2c3c857cfb02fac12bad94e53b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/e4ff6dd771d78a8d0b775b8c9688c150995adaa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/04dc21deaaa08eef5cfbc49a4738185b2fd1f365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/2a7372dc09b954aca02accba08d75ecb06d9d23a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/3ef1e13983a366d300dc05a673e2a38eb3c5be3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/486330f75a1b7c7928745eb944f29050c7acd3a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/50357234a6f6b945035415efc62db9683fc271da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/5c00844d4114b83e5dc6c430624bdd499d0be3f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/b6e415a7a3d0b897ee973b54109fe0a7751daff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/c5a3423c1e6068ac722cf532b89d37821199e6f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/1074d3431169a999478bbe77b1c8b4bf0d56e51a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/1b3f6bac3bb05012401c617d1f1784b0683e67e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/3bfafd8473b8d132971c5ddbc153f8e08c5d946e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/4cb5ba2c786df6f5096f862b6baf09422fa1b8a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/4e05554394c01e3f6e011758868c8c4e2a75609f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/8146a6e1eb72889f0481bbbadef7a88927a9e0d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/f5986abef2ab86d27bfbc2bd5069c0d9e1fa6779.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/fbbe26d47baff09d494a01db3a61aa9ba6a88936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/0daf42917a3b4d7426b625a29db8eb8e772490e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/2140dfa929411dd354366caf19fe47b9f190a88b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/69c43b0dc8217a191118f32607dcdda9bc0b6d7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/6f7b06992693d39c065b7793dfc59426459a3f9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/77bc2d8970f63917c157a4779f9a40bd6e41be46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/a74613052d0a3187753010509aa71e51324a22d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/d3130b00b58545c1647487fcbc07bba2c0f7fe60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/dc3e534f48c458af28f2a03a69eed8a9c992966d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/11389d79a7718769ffa3c8bdcc5d718004697579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/4cebc1bfaccee21d55c5298fc172d139b82d5694.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/4f3575006f37df01285ddc5215d6c39888f6efb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/51cf8f87185b23936696bdda4d2e5768bea9e55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/78069270885a567fbbdcec2c0817a3f2e7b31c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/9c1c09b43c10617521ba1eea5a5f6c1d2d2e7346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/9e84dae67705465b9d48e8386bf2c26e0ba32b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/fccc6b6348dba137fc36ee4f3ac8295b8f79751c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/0aac5577116eb4dac03039a62f877039f4924f9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/1d1325505bb15bcbb8d015d489969ee4a7a2943b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/388a2dbcacbb6babc70b09b66c7a5a348730ea91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/5c8325c0835fa69559f4fcddacf3d95017ec0f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/74700a9ac9fb9b8061a438715da6453119e07935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/9f857daab47c4c647a82161c5869e02e441f74a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/bfa42d704f2a64d3ccfcfeab77f5244b2a076a9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/f9e20ef03519c4d22868e846cd81d801c2c22afe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/0956f76c39f850460e68ca69e45df841d2dfd970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/0c060f1c794787d6a2d05248f10758797d880a9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/34b73aea736ca71e8018fd3503c636e7fe8a018d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/359c2846d4a7ab312e616b7534a2c0e917bef4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/61a722dd5a6d9de479ae327c22fc38c2411890f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/978929619303c38fa103eeb2d0172aff10d73f4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/a9bf471c87843621af00299b19f8ab2e2a76bf28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/e26e18959b8d51f21d4764a46bbd20e1e719e91f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/03ee48e55e27d5ac8c2676c969da6f5d0813d670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/212f942f3ffd796a5adf89edd8653097cf0d15ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/74a4173e543ca44db3ec428c330539e80c66b602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/b318cdbd6d96dd718c111b3df01f5221bfe45a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/b8351541cd87aca8534fbfd43d72f2279e8f4e1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/d9bed5383bb5245b62e2e32392ee85ff54b79557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/db03cb71fe67be8ae3b0b1553ed9ff10277b0b82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/eaa2db4f6e4663ed261c0bf033fb8808be814d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/12974d96ec39d5e4c0e921e5a854dab007ffea39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/28903ef29a623093da4b1f92265a6d382c44b4be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/2c9ab7db19437b381209ccacbaa408e398b85bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/38cf73ef19a78c06b754dd7facf8596d70ca42e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/726a0d904a1c75e76dc7e5a8d891c6e4ecd2a3e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/8da4f033b5ff4bd5495db3b428c71ed8e2c339a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/9a26c6b6da59c23bc84d3677414498b26e19cce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/c61880f07570a43e534f6ba7476831dc9cff4fb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/2436d153f78fa74c82adbaa8f26cdcbf57cfe8a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/2b74b26ab631465b9e87624adf5e6b54b983789d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/2bcb304ed7537f284f0cb3fed795bdf85851e7e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/377b502e27c3a458cd5a53b36a9c75c25a49613d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/7cb92c32964ecca0ef3178ef5e10754925289c31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/dc031ed44677fb0c607c181f11c45d9876727467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/f741d948ec9fcfd6c0e2b04eb8ae5174efe524df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/f95ce42e6d5c5bdd4ef30549eb4db3f2c21a431d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/17861282dea7a03da0b67e62ac97c601e46352cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/258cba13606954274f9afedb62823f5c74256dd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/42dc20932f2c3f641c8f41a4df0d7db61f6ebd68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/65d50f7f87f34aec6fb584388d3d741b4f88c021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/855ca48fed68fe11a3d0db1e1b7c057fad53f49a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/9ca617d3a02fe0213b7cee44bac795f0a12ff93d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/a0cf1efc60ef23b619ffc8907bf01f338de91246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/fd45e83491dd54c5bf7d85b2d251d06049348748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/0d538a484a88f48d5d8489d85b83a6cbac102376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/1123626266e3217de05efa6aa919deaee567c0fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/1da7cac4fd8823b4341b533405d2cce20508d1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/28ab688a9803392454595ff90ab64804721ac06e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/630753614e0b612afb9b687bbe31f3809f3795c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/71356543f5c2435a230281d6f0e8ade5e37383d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/bf5236d25c63820dca99d4212cfaeabff1de4cf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/c41725fc125974d037678ee5c504ab52faa632bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/06076057e9838c16861b81f8ae89d75733babd30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/10e26ae2accc5ae1d4cee34c749f05ec59b38f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/14f09dc98fe28091f5cc819c792a1ebe2cfea276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/a1f508439cd4734c682e31d47cf3e1ef09c06591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/a6ab7bc49ae9b674f9a4fbdc1c2a85d79f2db2c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/ad21d1ef01c2d0a30f18ea9c2ac79ac67492c976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/b3966c8f63adbfbe90bd3f00d5b33c22187b2f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/f748da617d18a258d567db0e1553d9832c6350eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/41e32d76a0496be83a22565395db61b55631e8f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/524dcff6e95cccccedc6a9ce0c552eaa8d45fe33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/6ff21ddc0de010ae04b19276c7bb2e4ad80308b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/799b0014018c8e19375302982a3dba2fcb6b511c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/7d71631e9a8f304fdfb113dc83063a91ecb645ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/a770a7cca4a5803b5855cdd21a00ae7ab777ea95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/d93eff6c547889e8c6eaac7ec62c060454feea35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/db62a2fcdbb634a9154ccbdc4041b02f234e3b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/08d7ef21467e45ea8d33d911a8c822e040d1c2d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/3090b65c9fba033eb4409bf70f8244da8ba6acfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/3d7cfff847a4c693c4a409fe81888dc1a255eeba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/43ce935d2014e5f7cb51e1a82f3be90692b3ea5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/954bc99f5b4d91442fb0f22cdf49baa516e83222.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/c3438f94dcc24553bf74be0a8952a5060736635e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/d0adfc4ef1ce8c5447bfc3dd696765893c47e8f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/f73cba93a59cf120f14bda09becfef00ec6499c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/089c7e4509ec84e4c1872dc448714ff25deaeb43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/34f23fdfbd4b0822876a52b423e09462caf95f66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/5cc7f7b7b9fe013c7f10e4daa070c0e9ca605ff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/82f98831803e6310b2f21d8264d8ee0701a05bfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/8ace96e2bba0c2924b1d07321ffcbfe0283b6cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/9d985c1a6f79e92c5cc18b28b6c93ab114bd0d78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/ac7219d2072dbd9f8be4634281524c8a26bc9bc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/d1e28e91d107bd022890a4f49c54ced2fd320b33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/3327def69a6d600d61c8b5de3de07c4dd17bc928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/6c08d0c2707bedaa7096b2b110641ef35fb14cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/6d1d4e193e69aa603320b1bc73b091f4b8c0c992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/7667cedd86b79e51d0be99a35b22829f2d7cf5c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/79aaea82d2f464736ea40e70d887ce9400709596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/af328dfc5769d158d9010deeb368300cc6f73b0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/db66557248ae89319b2a88f6fe8f8dea6a07f978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/ff9576395280bfe9e42ccbb6fd4e27a48a31b7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/0875b5ce4ccb3227b1b09e1c52fdda22df94461f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/26770e16cea0276feb44c460454b2ff024b43baf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/3f01258c7496092641115bfa9fa22f221f0406b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/5885d15805a3ff874578a2a28341d665507b6ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/9e2523bc28c74224f18bc2302de1a966b53dd034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/a49b4dc62d61ecfc67da411b518a8c4e0a546b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/c1ab6359a8ee5308a90ff4a6aa595e0574aebe34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/f0460d62c8e48aec8b3c860b057b289bc70bef17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/508056e5028b9498067ab1fcd2fcab04df3f1dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/5810fb8829603ec8a14541c4a3302f4d378e4442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/6726996c0e86123f5c1ca181deb77f463a71ff71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/723ae2edb40d7e2234509bc640c6387819a55aff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/769791cf42edbb59edf95c59fe94a997b55750b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/8d178eeb6ee06c5ee40e8d7c8326ef0918d14423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/b960dacfae68bf94dbb2e01244cf7f2e9bb29806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/f2bc75795d1a4a41cf490d34dc83cc129fbc06ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/091dc6a834dfb4d896fd640c1d3ad90987197129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/45fcc71bb00f2fc99b21e77be592a770fe227820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/55622ac298d2f781ddcdf002e7e381401ed411c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/56905d33d77f3e23e4581edbbccfbdf131b694d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/65ad19336b7dba2e1aa4084f6cdf4eb3f6062638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/7064f70ca38e5c2ce812ea95da8ef18a4a3e09e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/814d86904423cedb13a9415bf0f314c9d75e7f43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/c97b358509c4faa2f1870c2b22378876b11d5e3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/0ab61ee11bda304d46ad307c08ecd0e6c4421b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/24d8039ca26fcd568c872a974bea7471bd1d3456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/356a86092397b2d27ccd9c667c94901c9961c2ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/424948d94bda5fd77ad9ba29f21461f6071c450d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/7371831e346915b0d1fcfeaede9c9ea84189a69b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/8cd6009a699e25563cb60f4a772b4e9999c6ffc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/b1edb965f793b929a911a3b3bfa3af82ae91aaf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/e960325e84117a9309b8aef5cd2eba9998927344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/2adc29051a726b429df79833ba6e7028f0c29f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/2b1fd7a9367ec16288e16a14dfd2103a5152950d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/6746f2f75c37cb8c1ed79333c1f39740ccee0602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/7afe13bb5ad24cdbb71915935ef869cb503f0b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/912dea92ae6e21adcb71a4a35b52eb72c2f984fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/baba1cf81672f9962b2d0027d322d08e77f88f06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/d5e2f7cef751e71697c6b690a0f2f4bdd762c69f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/f8a31844f29eaf258f37edba27c0a8b2097da091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/29cba177ee62b36fc3c43953737c3cf8bf10b28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/2a84ea078b820e908108951e810c8088dc077845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/714e887fea3166bad947a37a46ab91dca26d2f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/7f74ea42ca967bb6b3a83640fb9061b7871e3b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/b461392ad472f9f8dd47368f10aa38ed72065681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/d44840ca0372a745134d46bee4436577d6a7e50e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/e73db322227ebe3821b4a8e0c4de31939f661681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/ee7b85101b78616fde16b7c53869e791d7fb5902.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/05bf9a42fab9c06f5f977a7fef68c1768b338cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/1cad7d43e734c85b8106073c2c5611add458738b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/314c21e0f550fa2f185e0e54a5fc310f87869867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/80aecf095235a9551f8480c6045cb3d00a30d977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/a9cb3dd7237c0a95ec05d443f547ad3ca5f52bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/c9a24e47e9bf31373444d5d24abba1b71c199397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/ccd831d8e5398e8fe949bd30dea1819fcd5e9a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/f11d6413ea615e486c854014b78cb68fd1ffc80f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/6a89042706224ad98f039155094ec5ed3682bfa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/6cd227fcaca1ed0033e39dcbc9120ecd3342e38a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/7d6ab287278ea3a45061b5d9e2cfdad38bf5c821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/825fb75e3c4640c39f2d519dbc8968db7952785c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/a7976121d555d126d7b7c23ea15f59c67ee99eac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/b83e27e6cbfd9187544348b1040360feed9ad1e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/d26e4be1406ac703ddd92bea64ce682ad0339d2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/fe076922d9f30f24072db40a20fdb1679305322d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/188e310fb6b76e78f45452c44b2e69c18d83d4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/4836abf777f7300231478ea5df926c02efbf0878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/4f1c74c69b25a35bf772cfef0d7d1ae5988a3018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/9fd984beca485e88705d2e3323430df736fca394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/bf5778431c780dd982ce448ca655fefd39469c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/d9ae925cec5155b627aa0aa71b1a9d4aead8b58a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/ee140111e7803de5a448bf23b9be5b27ba91e18c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/f9c2d67ed93f8c078074ee0d953af6c787e79d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/74576b02a05deb04da616a342a351b4f5b49fe28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/83bd3ec7939f89f010b242858da856b14e516421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/930eed3ed805e199eefd9c043a7a15f416731e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/9d6f214cc55a8633952a91288b31f1263a7fc7f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/aab0adf9e88b0502d6c5afff8370a13b4f8c137d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/b4063bff0f75a05c2943f4dae7a54f32f9f17d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/b4c45d06c0d76df896aa1a3548a11c07b8512394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/e63d650142e236164059827d3281367c776baab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/27aec3179b7ed08eaf658754b3ee5466fc6ff378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/33977176abea3b5c52fec51a58213ea405d3d9a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/45397f9346105035dfc128901f483a05f7cc35f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/6b2be203f70d7202066cf55013f9a2a29bca1410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/8a5bc426cdc4165a781e3a5d0cd8535648e28818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/90125263a2d13434f141b5462a0b1baf56683111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/ae22afb6f43e8b5f275015b3a0a9bfd650e9f384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/be9cdb3bb5d2a9ca9fa7a1dac60fb87b436650e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/185e08e1a193a5edd2845612b2b2755a8fff7c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/19fd01d3b22b270b1a6181758f8fb87c1e33fcad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/1b6ac009ab46332e680325808cd94856d062ec94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/202cded9b8d9e885782a36c61ae0d1a6df971ed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/351374291ee1cfe80140fa0ed6e4625fe15d1c47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/6da0dcd252f7a4279ed4cbfaf16dbde8f5f6e05b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/852ecaa35348de8b875edba097ccf425b10562de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/c493f28da12320ff56c9ad6eee4eb74e917d5ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/0c17bb4e77b73aa729d92b5ebda0aa599102f98e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/0f4bd28464a49c44848c50d449a790d391b4d81d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/67ef653987008163595ef9609b5111145b5dd6cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/7235cdec3c7b463b148ef7e55f446b9ddb6e4c0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/a93b5c4edb109d842c69b7883781a907c49f320a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/c1d24e82a12f4a5ae30a0078cc150d7a76641d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/c936ee7abf5d1c98440c6ffdf2f7ef7bdfed5bc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/d3e01d06ab61d8506d32f35fa9fa8c82d6641c85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/2b3bad66c4f0cf6556f15ad509373b1eb1d4d74f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/3a298e6436dd90a3349b7b8f03b39463a46ee996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/5a3147340af2645bf6628656b6493922878a5bd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/693c21f68ba9e588ae2343839be56c955c59be75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/9a97de8983b7d4ae49223a754d3a074e5bda0a38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/b673b77d31293cc2eff591580200a93d2baef220.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/c68efb7bf015658e12b237af09f452d77cb46f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/cb8a8f55c71759e19ac0640a7fcc5f174ed4a1fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/092ea3a448b07bdbdbd1d565c3ddec67cf1d4d54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/3e0208867c1ff0551b4aac8d9e4409ede9387c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/722fc52d7650f021c1a27ddd2d8951a22be1cb79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/893301843f3adfa7ca7a5372bf51b58da49bdbbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/95d77396c69fbe47f6b77a391232c02fec1c028a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/9bd17121d3faeda979b4896568ce01a931f64ad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/9ece97ca052935a4cc7705bc8cfe2f2004c45976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/e9a5bc4572738b0263790aa812ef1fbd9f73f7bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/132975dc1b6a0dcbfa4c278d3f0bf9dc21a3332f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/415942a6d657fe4562038f56512855575799b35c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/758525fce31532c6fdf0df093c6f625736b41812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/ae67f84d74b0058ddc8c11601b96d6cf0105a4f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/ba2cbd7f43861740b38d62ea9cdd254fe5d0ed00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/c4edaf93951ee8d92279375325a6933e06ef6ab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/f1523f535782295af68ef753c951a7d629eeb084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/fc21c6f5be883892581311122821e9024155738d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/01f51a9b3abb870032a3b19e2000f2af7212a5fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/171f44ec0f43edfac68b69b9d33e5bc9fa91b8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/8c476c830df12838bb100b6ac84fa096fbc33ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/bbacb744893a741a293d652ff47da5512477dd14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/bf3e4f31339c2372dd21b567c0954a796dc986bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/cb41cdb8e9f05146e74facb87ab10fe11d5905d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/dece26d4700c1d637a1d458eae132d3935e2336e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/fab210ff17d3b5b065a2a50874785e035bf3f5ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/1971c94a039f9df4acd3bc5e56deb5e19628fa50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/2752cf458b2284f44aa9bc66a47127e3b77cd952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/359ecf243e0815f6e6420477af41e6c3846865f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/414d3e6aeb3d7c9eea00ef6d15b89082288fafbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/9d5d28fc3378db836d1b02933940b165c23c1219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/be7a09b1f3f1fba2968734443076cfc01eea8282.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/efab1aa9e1cd3f7309a4ab910a3ef833a9068ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/f24cdc8c178e221aecf0914b90b945acd4f4b266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/257ee85b2c351dac6ce95c4e7a3b104b230c61fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/64d5b6636921e78e4de3aa404cb3569a3ff7a636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/6ded77d3f2dc94250b6443a1e78e0f578d81e5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/7e682e7fa6ce0ba89219a66f72a3de5b7af19ff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/a037a04334fca8070f4fbaf9da998f088b15af4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/a832e742ef4da3b8e827aa4ae76115c463722b29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/ab8f417d01d53afc2b1a59b5a684b00ab56ae5c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/f3974e6a2af64916cf2e31049e25498226cabdbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/193191c5c509d48b843a57c71e25189f69ae2510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/287d33859f0b0a1a37c5e2b277aefd04e2751e3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/2b25db99c35bcbe4004152c663943d1a137ddb59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/5d5aa2eb48e76912284eba885eb30b7dec0eead1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/772b88a223d580bc7e1bee02bdd94fdde3adc785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/a8178293c3f3eaeb5ab7bb8888f2987a255b835b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/b48085d903707c64bfa24e70416fc0c3638a3492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/c5dbef091cf01bad2a1bb3e55e4068ffd22c56f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/3066bbc24a0bcbeac3598a9a2962b813ee72f6f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/7005b1a135fcaceb07200953aa58fe8f7fe0b732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/72b4f771134f4b032cad00062808d1218f4a24c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/85f368b5724afe09c4b848e9cc6b997cc4cff335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/9259d67ebf67e36ef2ba672006088dd1cf138005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/b724da4ba337105c6a2319d5dccb8be6e6469e87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/bb8f9f942239fc3577342d6c4f2f915250880e61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/e9dc0951ce84b9004802e4ef11602b7de0599f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/04034b1a4ed9a56b75e1f0e6874d967e5eba000e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/1296db1eff6bb38a33c76599c31436fe00762c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/3f03684ceabcff799c6081db9e77fd71eefb6239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/826aed7622ecd1c27706514483f3076a6d35fce9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/9183670938961f973d788723b6c189d8129f927c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/9a96ef7180e4f8eee211f3170ffbac1e32011b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/c7363077a6dfaaf0bd97eb734b34ece6c77cba2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/d288910745b16716ffff468fe5feb3a0c42753a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/19328b821d2450b3d749fd2fb5ab314a0ab374b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/3d447a02b6d897034fd406e20c3ca3247f3adf6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/5da37c5f5bff573233d75cc61908802f9a8dec19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/790ac471dc863d4bcab5670193e6529fbffa6c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/7bcb8cd7ec1e5c0ce82af1110a5ae0266cdd35c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/b038340e729ebcce3d2c0bd41b7afa25af6b4037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/c03cc439062b84bad6a58eb2ae7026b43b2cd168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/eb07ac438d2b5bc358fb4fa039d6c67d5eed5888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/057fe6d27fa2fcce6c93f12b1ab77bef2036e4bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/178e6eedb4306bff067e5510590e4c926c74fb14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/343bd806d3d016b653717c6c89a6364f8ee6922a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/3f9262851a3a93891052df7408cf9b7e2acb45e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/5951c929c276512dc64f908c2141f8d6d4191bdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/770384f11f8d77dfb16799472ee42a4f92a4d60e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/783f0002ed86eee7b850c9afc8c0324fa1a5ba02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/dc07964b9e44d079943d18812446a1f09277a008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/0882b686c01cdb5e20f59479da9a6b22be3a2bd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/21589d0bf359f07e36eb0c886252827f7f99b4be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/4004d4dbf3b69b3e92dbde965e9777d760a0ffb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/6a3e0536b541b9df0e532d5d87cfb25cfac71760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/de2f914a23cfc9998727d5421dc1fd687e4d3879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/e577bc47cdfa6edd386cfa109b5360a73b272776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/ea7333ea2509fef1f2c2454243f75b573cd5e06a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/f5c9b04859ff49eee0c6a48c179cc013a75d35ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/1bb37f21c85d26d242e7444e497a077cc6dad302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/1c2de79f0e3d11c57be5dc24c965bbf423f9dee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/6fe3d46feeb24839511dbf736434c4b1f1bbaa39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/7f9a783aae9f399a8135218c10aadd8d94e1291c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/9f9004cf7f1a8ba2d743a0cf31fc67d098c5608d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/cd273937ef3a95f07f35fe34fd2757d96d7d3b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/fbbf45f8784184bb185b56901cf8d59c178508fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/fe5fbebfee5055415f057f23e21441885c6c50fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/31cd48a8deb63577fe6cf326ca0f380d6f8e1ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/545a0108165443b16838a947410f96194d3e9152.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/60d27082373dcaa43d45611052b2e6f81134aaf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/8162a2d85a2da8b3967ba5608605685efdc4c080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/9cfe7ca3e71d5330f4e58e409455323f7f5482de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/cac396e5e86e792e0db15e754fa01d87bdeeeba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/e30c91d5ae8fe8a1b5386d8f2865cd4862ed6411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/eed5accb27f9c3c5300ee58e25c0f3d8f9d32333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/1c9384a60a91a1a4455ce0e720ac18d8be587dd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/235b63df211208c51c76c9a75dbefc4bf535735a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/70c46be59f18d59a487ab78c912696c8f18b26a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/aa0c8aef6bde08139552591a5dd0aea22127b7c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/b44aa2bc0ca3647d5e6c81f33ac5fd897aa91e14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/c79cc7d5f8697d7a2240c29182d2cdb6a78af066.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/e74a803b9fd30ec31404dd02d34ff6195c68c277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/fc894a79d2761086a74ea9ee1ccb5a9dd937c27c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/56386c9f9db022c12a6b5dc1cfa2739b664cdfc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/965a0a9cdbb40f1e8d619a820426b203161a82cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/9ec28bf08f3fca07a8acc208df762a08aafdfad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/a2ad8c31691e682948386cc32eb0952df5687f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/be5534e7150dc13c5c803ac62f40b1b81b9df891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/c3575b7350b4dfc6d771140a4750d3fc79b99238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/de5601cf03a2a7aeb0a18c23892d914fedf5cc9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/e67cebf92e4f12ced087ac5448cfbd91fb501070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/297d29540226a40c2f085fa7559b13f89a7e1077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/4712223c1e8b16a2eef0fbb5717aea3960dba814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/906e78d869987218027407b19c773cda131d6f2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/a03fe4e15a015f7a66bc455d7e3d86b2f7547280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/a2f7316a374cb4c6c30e4d389de50dc99de66112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/add47e8d75347b25f29ec52a7d7b11f3c0bb351c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/e05cac8e362eec33e5f0e619219266e14941eafe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/edc74ef9d6ed255ee205992e7e15e2e6e469f2df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/1c4b1e1c36d14f11c1c373d6f79171edec2a99af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/33343e4e43b13111b53f0639f2c34980378f6d65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/7ad2ad0b1c679088e4eb8ad509d1d30439a127bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/80b788d706e3df388d444d3fd373ad72eb55b804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/9e2198b4b8c143c222dad415ba2ddee2798d6385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/a888eaa8916fe5604446cb08fe2d79f13714908b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/cc9f4c267f2b5ec362985574af9520baca76002e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/df40b64a72aeede5b179803c499558a8aba614ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/06ce2978c496e60df87ec44e4ad94e7944763a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/3218c9c5f76e53c6199b984accdc2badc136453d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/4b0a0f48f77dd848bcf14c5a70bbf64b861073e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/54f11bc592ed6d5f72eb7659b1d1dd0116466182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/b3115198e328142fd25a0cab30178b6c31c4cdb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/cb190c46eb18dfc970447a1d5ee9c5ee458f30df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/ea8664935db4dfe152113bd65dd91de9be990d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/f6810ee96c555b2e83e144b79c84f05d664f1eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/07d614456f144bc381bcf5ce1dc75beb54f6a189.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/08b84323be0841160ae83d984f6c0d31881bae13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/1ccf2c60e916829b806c55e69e9026ff9e14bb8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/4ee5591cddfa6c5f352f8b61b7b7a1cca9a1626e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/b9395a38c4c6e40f5efa060f48983a2096319e2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/c19bdc0a580e37f5e3484688d653ac83bb27d4e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/ce327dd23ee1530c8a6c988f0daec83f13f13a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/fc73a0b4d4b0eecbe8e7809b7a3bd001dfad6f59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/1934465bbc52eb3dab26d8e11b1d3ebaac2f5947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/31647328d8b5f90aa84ac0c614003d7f08e5e947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/5958f9a6078c231ad13dafd7d28a34769e7bdebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/672abc0196b7a06e9e53c0491fc53c57b193fffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/b25e483d4bfb52f8273e94ec038277f1dbca8d75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/e6780062ebcc3b5562cdad4ecd56ca939609c9bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/e6a44dae3ffb0aeb4e326a8107e06117214330d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/f4bad3ff0c27ee76bb5a8a68b7f48e8802862800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/198456865d87d18e2e9ba4d8303a9a41d4590486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/2ae3ef436f951de44683621bbed7e4bd54e917e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/561b61850014db3de230bcd82415c2bc894ed466.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/7edf0fb1c82dca16813bb76bc2913c8133f2888a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/ad5d258bb14a9115571535bde5b9cbf96a01be0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/aee74235894208f2efa5ba0ef491784ca762ad5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/e2feb1ed541727666c3988a2e3e8ca64e9148839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/fcc691c168dac581f6a1d6fa5ce1dec087cd2257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/2009cb4e8dd728ef71bb1d634b949358cb5b16c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/4f5e91c5bca8533753e72db984f44e46de000caa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/5b38609dbc96e33d7c99811efb4f39e30872b162.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/77c14d0e6c0e074c6da908cf63ac4be5a77fcd02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/86ff725a19cb2330c8277caaf769099136ff98a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/a50c9d80f8e85a161da7e9793df9cc46210bb3ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/b68e9aaec2584ccf2edae54b3bc6bf8de8412d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/da22b146033ed8955d0aae53c31daf4df24304a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/0a741ca30487a17f599b18d04fe8655bc679c0f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/2b57ecb1760d1e989a8a647deae919474ee8da76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/5eeaa2ff96093daa17ae5e462082d69aafe6e1a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/621bef6f4d919b97fd3442cd5e2482613304c782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/662a099230d8a3989f8953f77904aa641c217d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/e83a2db9d880bf5411e044e8023048bc5cff66fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/ea2bd33ebc91bfefaafe7d5e2df33ac2b902c281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/f36c7d929fa946968e83fd0cbabbec2af7645c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/0aa4a48eb5e749231475aca0b7071d5efd859f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/68dc0d6028550b8ff8f6f169416ca4b4246524fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/9e226b420515c4a19ddfa9b19fce73a0bdf88316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/b0415fe61ba90b437f7814f4748c5afec1951fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/d776526b5e9ba26debfcda04fb50563be73b1302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/dacc0f8bbd9bb154180ce35ab8044738ea22080e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/de2c9bbfd3c30e2142d45cf6aa0317b2d0ee8b3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/fe7adb151db84a1535f14c5aa233c1171228e0bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/030603a457c354b140148988c1aeae90d5060f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/0a629e9deb3f210855b6ed31cc6da85197afd739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/1eeb8949ae5398d66e11b308073845e7532591ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/40f8af1b61313a4eadc84c2dc4e2414e873e3617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/4988d74e4ee0a1a590f6c2191df35f88427d79ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/6a3e121f18c9f86f3865cae35b9261e571835cb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/a0876310e563d89b1a00f7e7a8c39d122e2b2a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/c92da22c1600de8162694cfa0c5e0f434832a9ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/0dd223591e356e6ceb3bda9af84950038c1c19bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/495ee1040fef0d73d0e542148b5f47599c6e43e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/4a3aaff7a285b2070caaa4c9f98f841f06e3c294.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/610c9c40e0df85f63b7b8aa73a88dc4500783df8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/6ebaffe740e948cf0e95950217ceb8a601f1a333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/8975a484825d5bc9b32a115d3d65238482a36535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/8cca3e90d0a23a493e6679d643cea9a3fdb345ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/ad8bbf9aa3df0f07ee518dbe55c3d46552773f0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/08534e3c22f7da3d799a06eb38eb679e0aacb5bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/216cdf622e1653241f2e305409465ef3a3fe56e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/4233392c46f828f81271d3f1236b2fceed79bc7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/9a17a929ee7c0ad83e34328117aa7c015ef3f1f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/a00e52e2159935cc90f6e3a385bf535fadbe392b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/a666720f325801c90f6d955dd14e26de2fa0892c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/c602ebac6e109e00c9cf7481592849e66257245a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/e08cbd8ce795e309ad2a621b3c4116769286f7a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/02c47c36b150f60c131e9a7e97eeb91421783187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/062ad17eb2d969e31aa13d871e1e09a3b5f3b07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/40ac81ac2170cb3132e25e33e07214aacc334efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/501c7fb86629487c428b3fed9256e83988e34a18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/5860dfb375ee14cc4fdc7bbac1fd68f61bc82758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/8bca0a3a124715ecb76fe0a42473bf5da77adebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/a982d7b7894f0a8081a9652d27a6771ec0d72af0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/ddb2d8f3e7b458e0f96b0b6a8e62a445672c8dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/4f59c9fe0fea49114b7f37e91a19a8a1b6f01b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/556b77f6399aef35e575affaf385ff2ba3e033c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/6568de24861fb658b0d211b71e36603af1e0e51a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/89521168d9dd82f7a24ec5ed39057c516ec99321.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/92788b2333090a39fc685e5aca949d3c3f59c985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/e13196f736b0e0c099ab608319ffcf7f28cc0287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/e2a37f8f2932fa01bfddfe02fbad7d11dad837cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/efc0c1a6d3e4164f22ff097ba3165ed62d1e8a4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/5ea473ef2127698bfe229ab6b25b361580354ccd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/72e8047fa932c06b0f37ead494dc364149e4c84d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/8a2be08968ee41b0b6f7ffd2a19adbe44bb3d3d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/b8deecf451dc5ff89b43b745ee1332baf6d215a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/c1047f52d94db72dc0fe2757afe5165569ff8bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/de2d8d1f3d479f614fab884b392596fcb10d7ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/dff65b0a615ad991ae6fe93565359e254b18fd80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/f876160641e0b57052f6746d827e33c529377fac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/7b15a73407cea9823bff773813825ccd21228e2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/7e9881dfe829e0f82f6146a3b934760c63529806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/b9232cf2476b69cc3e390c2bb0364977a5db8df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/c8bd6599d3b89f47c36ae2f14fd6fd71d71f7302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/cc3f833eff3d0a24c0d34681599b1124a3cc44bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/dc75e9bed842bab5d5716a118e5208fe781dc995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/e58a542fb9d48bab71a28f3fab810eec53fe002f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/f8445f2b09b95e2ca7a49e972aea8ecbe27f89aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/03d6258ccd4716cc22045929c73843c4e3eca0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/1dff7474c52b50d66f081848fbe1f3812d3fe564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/36bbd795760b9dccf3ca33c998e1bc58159c4b37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/76578c8397a21e35c8384f28cdf719bb81b9f4c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/9a82a7f0fce2068dddae6526e480bc670b424a28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/c5309751653cdb4f599945983911d689ac28e6c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/f390bb7e21c974234461b646bfda077ff08a0ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/f638b1366f769df46059db9b79b8ea6513f95031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/169077be7e96e8e4fdb42c63011c949c3fe4cfe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/25c8d05d117c30b89d54e45a0b4da03a1ba05e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/2961a3bdf9d844ad6ab5ac97226b94291da8c23d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/55ae1f5c858baf83af1ab7eb83fef90e56d92c12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/aec44f9d69718949c9892be9bb6d832049ca91d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/baf65b4c7797caa9d6c1d81cb0e548cfb4fdc076.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/f0a4af5cd7c0fcf309a6083df20e1cd102ddeaf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/f11be572a5d0a8d21912ab9794899ce8e51dbb5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/2000173dfcfef1a77be8beac36af51feaa80dd0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/3fe3fa3956780ca8a87d39e8aa67696370645335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/78366988236cd6513df2a3b3ad266fdde74be119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/cc4a895bf86e6554d26ce13b7ef6d1c53bdf67f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/d97db4933c6b9169430411aca78b2e7b974fbde7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/e09df729c93aad6d0b843bb2369d4e91a4a4b333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/ea9fc43d8cedf1119739d3991d8f2cb6c448cf27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/f81e12faf4b9ac0cfbb27d4be93e75d9bdd498eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/2418aa7d1a32f8a316ae848a3e86cecee11bc3c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/295ee5249a32d10efd4ddc989de9f27cc1f3c94e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/4f2bbeb43383c6aae0b6d25c6e822c2bc0f70f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/52fe1c99cc0de715d9cd2d4e18b298bd30e507d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/6f0f7696c76289bb74a478350582c5a0381dba45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/b31edbeb1f2066e6ed39be5596feec8b0a1cd906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/eee268735833eef5f31330bbce5e9d4b580c4e01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/fda7e17b4a96ca881373e3cb4fa453ced4412cf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/0f89c293c86580745719fea71779e2395ec23e1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/17569fe23880c3645fc764c245b87d725cbcad90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/1face8dd16025e30c1e4842f8531a33cbbb36c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/33c69a8dff9159b9881aaada53219b4ee29552d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/a47b07a569a66a1121a0d14559d674793fc12d11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/b52de89dabc32cc5c6f3923c6e43c07b646938fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/c350b755f6608ce96a30528c672db894fe4fa2e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/f352cbae55f91ca1506ad093750658b1f4cec0af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/22025c3be286c82420d4a62e9c3d8dc7d50b649c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/36baac9f23c71cc8a4dcd9ee2643affc7140e8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/37b179529a48f5defaf19e5863097c20e1719dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/3848594dd75bdad8d80a567597ed680397b29453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/9ea9a324e81bdde3f05dbb4079acbb8b32e537ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/d96c35415fd99f1b4cd5cac36f3aac88ade606ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/efe64aee74a1bb043080b044ed901c77dcc8f996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/fb846ecf799cb9e8d40eee1e5d1fb7e37648cacb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/157c962b9c6f69375c7fc025ac051838ab47cb5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/355605ec483c4e72d38bbb7881943990b5c4b997.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/3ceb065558877b3b93dbcafce3a1492f3aa87c2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/575421095a0b49b4b4ee72103ba6fb343d85b850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/5805482071d8b75c04136f5e4dc07bc497abbe62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/89c5ea0eef82aa4f5415d1fac34ce78d58796037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/f29aef8f57285b6bf4102632533bd017d2cdfac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/f31f93cb8783b64b060e80b96ce7b499ae9c8a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/1524d1964adc9ac9ad5392d533a4b5d3bd5afde2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/18cbec20742065f00059799b5f4e8b9078966427.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/24681f597a1f13262ef09d05692db2e835f88cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/3d2b5bc2739441a401aefdb2f8cfb873767bd005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/a6d7ccbf2ac5cc9b59a5a4b29d9da3422018a9de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/d0866f256a14f2ee017c1108a45bc97b01b6a6d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/d86329c751878f31446ae91f0a5417124fc456d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/fabbddbc298fa938ccad1aaf07bad580c92216f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/19dd7669eb1149932b0e4db4cc146881fa3f81c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/73560adda84f9442314a5a0f5f8085165ceb1cc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/a49c458b631ad40a6404ece26c84f59fb25063b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/a9ac66e20b97d1e7d9fe8366b4ebe780626bec97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/c46938986e4bd05759ecae193c0f3df48454bf03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/d066559d85df4886b6fcd1e11b4b4c4f0d9f8d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/f2944e38ccf43ea9fb92a29a85c1db0a296ec411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/fb6b152552c14c143d0488631ca78745af0f4be2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/8eb17aa8ef0458f7d8a6a377074ade00e390c414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/b1c1e0e82421e58ffb3ee134d19a344a9d21fae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/b7dd868ae79573bb05635ab36820f8ba03c151e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/d841a84e72d7ddbf3afbba3e8d1abdbd515e4b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/df3d9e1afe94c7cfed13d63728931acf01fb20ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/e61357709950e8752ec0154e2c8f5ad8e8eb1a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/f4385281f7cbd47847fe6e2ccf9fa6ff3b1f1761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/fe18ca3f6a3626208a059e130638e36999254ece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/16d003cb0226932ca92f42d19cfc3f89780f0e61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/17b3a993a2063648fc834d408d2a242e494b57e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1988de9567b32da2bdb2b13dca1c36a9f275105a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1a1b4bc8cb4080aa784dc1668d821e29c48ec539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1aa8abb3517000baca0a8d0ad630991d9bdb53ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1ad29f4de438a08db5d629a876fc8c020d0f2f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/271306c1a49400561c784bbb82c34c4efec48c16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/28f4135f0b647b1664c71a61d3886852c5072db2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/2ee1059152b6a7791d4835be3901dba71d9a3983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/2f5fbdda0ae54ce790810005840c97ada6c850a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/3104aa44ab763afda88638e7c7ce83437e3134d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/33bc2e14e511efacd8b3d7e7ab51f9c86cc3b94d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/39b757e117763d05c2cbf8d06d7e85891614e4db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/3f249cf7ee2079e32e5c378588265eac5bbe2028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/489cc7ae96fc0acef075eaecd87d8274382fa19d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/4a1c3c7e223cc485c5db4f9463a110304b7a91a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/4a62bbd61fb87f062c2bdbc3b890a72849218df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/51226611155ea0d39dcb69d795072b92a618d5b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/52c4518f1029538ee598679a1029c8aceeea5454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/5e8ab91761070a974944ff5b5defc7a571bbaa19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/69cbd42f76ca7d2a87b4b34d67509a1e4bce0011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/6e5a5542040bfa9dd6effe27c943d78bdea561a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/6e8da8c24cbba3fb13f091db94e81239f96afa05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/6e905a543e5263226e5f969470f7c26921a40311.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/6f66511d6c8e9b20486febd2140ece6a11598b25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/6f8180849b9ab38a7b9443ba93d870a2abb54b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/75924e8922ad58408416afe9bef2e35d7514d95d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/7685df6c6d213e87d767341f02ea316f6ce28c5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/79e699cf783879c7ea4d8787dcda872850ac31e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/7d40a828747f19b5f23ff97cf2cd71b637c7e8d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/856b055fce98dbd6505d622c854da18ee3011812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/9229f173245e730e668a9e755a02760eb2bda17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/959e5c652e903dfc3fa7ea8e037a00e47752f07d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/967cb848142868df0c84c24d02892fd4452512d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/9a927237cc00777ad81d98d5099f390801d09550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/9fd9296cade556cd2159ebf075dbf5a18da30a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/a69e0d047d7c248b07e878b82ed37be9f6fc9108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/aad8eec8f60493ab762980f9e9db24e8ddc4a017.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ac6ee4ef701f5de38a97bda9679a1f2c1962c7a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ac827daaf72919dfeb588454d1930963ff1f163a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/acb2798f14c55c3650e29337979e372006aed1ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ae00092af9c08b8caf29dce7905cfc286ebdebd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/b2d30fad28c8e9428c26eabb78395edd52ac7c50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/b3c63eb5f23f07f13586f95978322888371a2371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/b90cd4292dc7721bb043f2c8cb95ed340f66d361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/bdc951cd0350a704a176d3df73e54521b18d146d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/c26448c1720d1ed6d9663ce05a27dc3b3ed88b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ca0e6bb28580206fed92f1799ef2e929843f513a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/cdf0a73e2991eae321f1bddbd780d2596829d370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d1f1c4a979622a301e84672ed8b8f8e311f29a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d32aa7a31953b47fc649821ab5f66f7dae4ef1e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d436642be2842d09e9530fb5e26fc0a39ffcd2a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d5183c4f5c6914659a789674fd1cfd447d72e349.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d6a1b6d79b10c14ccef8d9c5385d789b03f6d4f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d809e279a53245e5b9ac0beb07e68a21b8ea9cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/da582cbc304f3b5bb37a64e50d36e8c4fe07a457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/df596c7177f7a5dd346c7caa3b4ba4d19ea63d62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/e2b79f66b98df773e09edc7995fb746e4ceb2ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/e2ef3f68d3c10668a1ef7e08e411a5039c95e14f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/e2fd4c19760b53cdc387ceb3d48f18a8a144e988.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ea39181e3a3c9eb4d9d7abce215d5185e32cf6c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ead6ceed24e5207612d53714bda9287b1c5f9d8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ebfed65523f8855646c9fdaaeea83452a8bf4887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/edb50a8498ff947d705d1cb27683c65771866f53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/edbfb1c1875cdbb9a77b215e518204b8960e639a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/f18560f9c88e045e0c963157f6702c4a8da40416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/f2b5182da549d2f2087cb0527cd444b2cace2515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/f67effcc7471d21eba022f828628b55685f0f07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/f714810ed8234626a92e24517efc42c1cddcce08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/f7dfba0e73a8b86d5f101047c20979a13beefdb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/fc0cea8d3dc635bd40bbf4940cf594536125d52b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ffe9b4abaefa4f4aabd5478b8e524b03949e85f9.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2023-06/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2023-06/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2023-06/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2023-14.yaml b/configs/cccc/dedupe-url/CC-MAIN-2023-14.yaml new file mode 100644 index 00000000..61a8a369 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2023-14.yaml @@ -0,0 +1,821 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/07e07a2b2877fab925cc7508c967ae7c1b804c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/2d626a929dc6eeec0acf19107575931e8536dc88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/655bb8bf55678acbc563575e3cd086b61bffd49a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/a1b0580d390fffcadb2249ba542e1257ef856128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/bfb2528ce1306066829ef2a8bf79a79fb2b8c161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/efac43be601786dff11702edf5824abdde183557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/f1b8ac202d071d6ae4fa4b6e363ca23b24b2f5b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/f68471b10d6059ed1f45ae8d0d210f46b49891be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/0b611d8d4c5ef9350a6ed677df69b50cf17bd7c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/49cb795787d32c334ee28d92db3a8a18530084c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/702c37f4575b996d62121101cbcb8b4bd6d99d09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/7c19e217e87cf1e1c93783780c02e42cea09fa74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/8cec3ad2ebb19b87bb4657024f5c4bf3bc5a0128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/96edd0d94fdc75dc8ff8143caf8929112fd4892b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/ac7a62ce75e11f648e14582862e5d48eda0223d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/d4f8789c68890c6836fe1fa18d5c2c8266e03f70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/3d9c963d28aa2e12cf9cc994c0fd614155b08c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/40b76356eae0336fa5ae68a4c7b0e8ca03096d7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/4c6fa4fb63684820677ec51f2f77bc54153c6f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/683dd5d1ade091215cc8d292b4c6921429b7b7f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/724d41942b9e95fe5bae7d912fd29fdd92ee25a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/902ce9ce4ce013ee93e9e52375c4fd9eb90131c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/df4edd0eab8a789dfb2476642fd507c43e6d092a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/ec5b70bf14cdd6c5483a1b4058375e2fa6c85d32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/0131715cf6a98710d300c2f5588f551816050379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/756c2f91b1004c78aa8b8ceac3e8aa424ba3ae0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/8e1dfacfc4b2a4b77432eeb0bc792887d5f80b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/ae425badc76d85edf4f6d337ac8d2fbf9508af06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/b1563090ac9271344aba19dab22e2f15bd06766a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/d6474fe30a44bb8429c16dac7600764fded8c3f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/f0634ba0d494b8aab8638e677892a875a123935a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/f355d187398f1ddcd47af34e675d695243275217.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/2366872b016e86bc329b0b228c7dbaebd6713c85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/27741b46945c3218a9e5d8970b6ea7d1d5590851.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/2dec45587609d0fa0a7b5d9f90711ded74b070ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/39c7d692fde1ff56170c149d583331e82ce1ebbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/8b324059d535ed70877e8a9b0a8b3fbab36c4161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/95091cde46bfaf0c2573011bcc7d2f897ae765f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/b7d4816e2b872a92d4ec25facfb0ed7a281d675f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/e0f2cb6343f45802dfeed81074db3b99bb496c48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/013bdb0135d5731323d97d2c4fec35b5915f9bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/4051236a8fb91739cd1949a6abdead4c9291bf24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/457e4fd9dea9824ec2d90ff4c8c7592323c9728a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/838f11f12e7e7668d03178b51fb726de18edee00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/8a69460cb7b76322b3554db698a879283491ef4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/a1580eb3def3638b75854ed74e39189ceb1fce92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/bd774f6ded5381380e55374c05876e036efebc69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/e35eda7f3a7a555ae087e0785b782fc51844973c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/2bd421ebade10253a94a0d03133dfa03155a9d06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/339e006fd80c59269f5d577c97decfb94b93e867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/6c2016f5eedff999b524252d0e26feed8409c479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/6c39e70bdd4fee2d989b7cab633b1b9483a8031b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/8b038c00a511a87074f16e6626cfa7f2386d296b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/9b27fee39ad431f5a680cc3b50b1d6c361dc181f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/a91f5544b25f11f8df9f08eb1284a10a751924e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/f26fdeaf87764339177e22aa0f9843d660e6937b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/18b7d8d9eddd092bdfea527718b68fd68a93ade5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/2c4999f808b37b6796b1a85eacc5956d48d12849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/58b7aa7d092f3915f1887f07804ca9a04bd12740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/b01868d98c25dc2f77d490925061d7f137658f56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/c19b5ffd149de5f066ecdecdda7f7ebd4c05a6e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/d48f3fea39ccfff592fc7df073236f4c1374e4bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/e4e785f672890efade0eb337bcdd23b16dfcf236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/f145cc872932afd88cbda5efa45f66115e8fe2c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/0e6918c21a32dfa26c8289de5efbcdf2f2afa75d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/2c13d75245a0896d7ffe10488fa4cefb61ce91c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/4bbe9367e382c56409790d96ee5fe9c2a2ac32ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/5bafca1402b48b5d2fcb205f9ddf39e06de5fbd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/7457f17cfab3ece9a2dafd9b28d8e5a5d49a7039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/9158e42f2d82ab3b5f84085ee00e46dcce9fe690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/9403d76b202b249c21f6eeb7ca51e5da1c859281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/df7d85344fe9295909826b9429ed3784c31d4d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/40f5a4fd8a3f725be6bc4970569ad9be4010b86b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/421da7deada4872e026f97288e6e574ccc7ef8df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/45b40ac5b0078a95190b021a2ed95cc9f99d35da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/7f1cb178607ad370f88afc885f35dd6d01aae927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/867cfab926917bb58b21b53ef5d1c34490ae9004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/e4143cb537449e23a2d49b8ef113ada0eded36be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/fd944e3d1522032ad71e8394005269c30b31e230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/ffa97a5d8f10a3850fb724c031864298b687683f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/30845e9097e062589c642673ad5ad42e4be26a71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/43ae698e13290b53d00599989cf54d56326427ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/441e14c0e9ea2fbe27f6b4437aebf47390dadea6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/57a2be42f8aa06c4d3a13a624c8a457e0cd67313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/5b237dc6975d699dee1e70ec341d72accf5daf91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/910a3e020db321435613e17a15b4d8bbaf95f857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/95d8e536355622453cadc6fb6b790de904f2077b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/ca7d2f2b525be25409bdb37117f5b2c6c196e407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/12b255bb34bb7cb2860dea55b9b2ea571b569c25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/2c34b2821fbbf08f54c560fa412041c3f0bbaaae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/52b8d71f620441c32d55f6d28d7172b80bf480cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/5566eb1b271f97719a3878637b1345f61b3265d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/a8d1965db710d49e6592beed45004cb0e67fd23d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/aad23e6911ec8f960afbe01b8d2429e2f5538deb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/cad74c6e17f871a8b50156091d083c4dd78c5ea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/f06bcdcda45cfc1615241ea43486c1e4c3653b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/0a1377419ac431608cc5e351f60aca6087a5a83e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/0eace66ad777857e9851c3c5e4dcbe7f44c10dab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/287cdf89269dabcd15b585ecbcc7da9b06148dd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/3f8ae9022b70f5186033441fcf4800ee78ffbc2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/5d0d4b11b5f32b64abed2c0208b40cb46469d647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/7e2ea35faafca256ed90a5506a843e29cda44ddd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/fc25c2a210e4524b309151ff12b7f0a7d21ab94b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/fd1e479eb3f4665643200d9e9f0d0e7761a41fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/02c096064fcf5bb90e065c9ca400bb5ec9e9ea6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/151ac593d32dbd7df8e83455474bb2fddb46da69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/438ca94da900f39b38c27e2e687096ae20874149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/5ca2aaf70457ece64ac60bdbbac788004e2a5580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/642df83bf163332caa5a8b554353fa758d867f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/b48e0cb4c1e63a3e22f4b0cdfd99d105a53f6ddd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/bc3f6770b92889c412e66672361a01dce7da4340.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/f0d2cc5e4e451a398820fee4c130591db0c00870.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/6e74fa99fc1dc12bdedde88c943e601ef919120b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/70d5bdc17dc519110907cc5acd31135378c56eb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/72eec84209bd13d562ced819754aeb3958c78cbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/88a408f5a0dde8f9f7caa25cd13f0ea4dc462cd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/b57c0387d8a4050509da9c6d70faf8ff2c93c2fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/bd97441c116723a98c7af4b9e25673c7bb2c43be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/c6ed8b926368d8b866de506483bb614e474efa5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/ee70ff49bd4b0a1714821f0a28e6f3db2558459a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/12f9836261d895cba6ca1bae84794f937030e2b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/435d743a49975d50fad04b022e502f00b2f1dea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/767f4eca5a432355493e4d90b5b637e945e18a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/97e2d550711330ae72d2583930bb901e4ae971e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/ad9ec5e202c46b0c4aa57e75e3077121fa723e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/af2416631d2e7c0a63d39837cc54ea82284bd8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/b3ce9a5285531ced3db594ad63dd0c7b536926c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/e2efdb081e44db2d8aa1403f79964504fefc6173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/44b830c4e12f5a8a00a4e00d0e732d20d7235a8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/4a46b84dae54ac052ad1fded9c3bcb20e600bb25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/63c5f63b7e894b95e520a7dcff3405f0b77adb81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/67c8902ec888815a893683d28f041e465721c5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/753ac8d119feea3545aa0c1c4f69515f1cd6a8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/7852bc39dafabed8e2103affc5d9988cdc5d5bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/bdbd160edde5c29caa88f01658c6bccf3b928676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/f7bdf5c3cf6e5e241a0090b1abc1bd339172b5f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/2598fc45dda94318fa83bf5eaa4830a5b8aa28ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/2ba5fc725859b9944ad0c1f49c24f968b602db52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/4b9ce61529cc2a6f6f00fbf1485c8fcb9be75916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/4f2be116fd65ebae99986ab4e928eb7b73a88ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/5cc1d147a98fd76f88801d202e939be826ccd505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/63afe5c13789b08eed4e448155bbbef4e4b47f8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/7dda76050dfc982d55848e879d7a28a9e8265e38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/dbc88835468a65a5b30723d025df86cbb6c956fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/05b2ff7b0c909f3014602b480eddf45a2227b287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/4a389154047ebfe56ca59d455d608d980bcb068c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/57cafef6e4998d1439f1a17f69db67a493905f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/790dd780626accc591ecb6f839564a35615bc557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/960b0407fea9f8493228f39324486d1f12f45a68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/b8449f74454f353f2e2e05392e05978335b1cfdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/dfc2b0e31612fbf441e8ae6e82f784566788758c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/ef098614513918981e781b319dc2d2fd9f4b9f26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/001baf1483272294068dfddc222b594b66bcc6f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/480e8e82995844388b39c71e6c5ce209d482947e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/681ae6e12b217e0d577e1a23f68092a98a3f7e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/6d266eb5219a979ac38fbeb6bff0dff36f105fa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/8133c838f5c05a52934097775972e6b40777d5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/a62284d1733fcbe9ca2f4d2367585f146e770d23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/bc10a25f57f6bbafcfe90aecc630c2bb5948858a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/e1fd5b2762befea293e0992fbf7e27b767f1d957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/042e8c27555f49d45824fbee7652db5ae9fd3d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/0f86d4eb9ad8f99e28968bfd39e4a0925c50f913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/4426ce2f8fe1164940acbd1d900298a792010999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/6f3258caf8065e0e0140403fc11084e24908ae81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/8c446ac1c1fb08eaed7875e7d0dbde4a65c8126d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/aea40e0dc152dbaba381ae598bd62a43cc09c364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/cdfbaa6768733492085c9e765c2af4ff9bcc886d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/dd8dfd038a2e8f424c99090ad739d780bd7f450a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/0f2942b75df94e97a21a5abd1b9abc348c5a39f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/12d807622942956da44a4f281d19d3a90d2548e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/195996f77cb1151b494a540c727386b08b587af0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/427fdd3e79f94123b05f013fb6aeb582a19ba5b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/49e4461332e5e7aa1dd883e37d41042dba6985c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/4d596bd6858840de6f72736bf7e2e0ff439cfe38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/a10eef119ca1a63bf82bbf0b8f240e9545765f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/b58e36df9aa70d00f61ff6a190f89d1f5e70aa2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/11cdc9fb2c4360ae95f28e67d83ef543aba87cd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/49823e2edc5451f2271f1bee2663efc6268cf0c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/7a6d8c91e3474d47c485db52c4f2a14f1760974a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/837c65346eefc5ea707999ab0d4f5eaa6de034fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/84707213d99887c3fde7d8d789e5d71fe72ed322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/ab289be1da2461631d241726e44c40f760220760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/df03c9960e2dcf2ef74480143730d5c2272afb9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/f9d2f68b24f8f7a21e50584acf778d975677eb57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/163a98bed10aa41abc2526beefaaacb5e855401a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/26aeb288e4f28f65257c4f2ca8b3d15cb52bf413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/42c688f7a4ae1ca20a881d16f40d8f3970b0f365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/6d05b3f5e571a70315d4a6b7de693b4395d05e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/7a227de846d491ab5f7fc0a6bafd40837b03b279.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/8932cbd81a15505f54f4597a95f42f99eb611687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/8ab7796be79e6bf3a5af2a9450f8d244117f3006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/f512ab090d10eec778a1766511f8cdaf7a1d7f66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/12ad902e29e22542e6f399e1ef395730b453b154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/27056a609acdc39629a27ec33d051b9d1c964fc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/2c2ebccfb5e664c733bc5981ee8b067ad8f6598e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/7d94587106331e050db3370943a9a26b6034bd19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/812a6d333d1283b7192f13f18cfd8da7a73585b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/8d504ee2345705780f4a64bc41e6713fa1c1809f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/a00e6a1b2fa961624bae9bb8fb44bc2ba02a77cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/b4e01083d2c33d678e128c3d89c520b5310063fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/194bd5353a38386d235050b0cb4e0cebcbba93a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/2fa616a1840bb0ffc189103d04622e661a1dc4d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/373cb439832aef7647e5cb2d1cffacadf13c6d85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/444a2b22bf6afb8a019f182f1399dfa6558f50fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/705b5dd4ef3927a6e7b807cab38d304ac9125efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/74e7669bd627ffd99e9a5c54afa28d6a8381f62c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/a415856b34918fa904902620429a2e2fca8917c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/bb9c3910a677e2b452b9ae3a2666ef92eec24aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/20ec367dfd45ca02165a5d0012f9df3fa1cc5669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/4b0e0228cf23207a19cc36ae57f8ea7f5842430a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/4b2ea23b3d359558392963bcb5c5fe1398a9f093.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/86e61ae03756131b6e1351b31cf97698fa2c5cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/a9e61b2ec6d01dbe70b04983cdad153987c25316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/af0ab375311de1391bb144e4f4c494fb63811d4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/f74d66c4077e70d4b8cce67dcd7dd142f0948a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/fd9dad866b9c9918adbecbb7d3c3fc796faacffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/0679d21683b457b881002b4a48da3e8c40e3b3bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/099edef03530b73812cc885a7210125e5f064c88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/1dfc0b81f6ec5181fc6e07264ab23c3c6ba1c130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/3ed2b377d5c023f29f5708a34b9fc420992eac32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/b7547bcaeb6178cfdc229c4db818f2484c31c9eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/cc77796525edff98254850d8086e540f8ac0ff1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/e898414dc0a092d4bb085ceac994d868a04e9856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/f501f3c6d58526e0a11413f76914a6dca3fe8862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/6083903e9962027625d9c72bb0c000af1654f502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/870d27a403317a19d58290a1edeabf785a56cb6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/8ddbfda5eeff3af2e92f841cb2573a848e6b2f4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/af33103aca71a1a0d6f06ace442ad5eb68e30565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/d453953f7c97486d16075fe70b4e923611dafdba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/d6e18b000cad69de4071e8c30ca43c4d901158eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/d8cce483f9758928d7f27728f1d055e4b3e27bf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/da4a3964880739475ddcc5b9522ab62416461504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/0b31910253f1b350084d4aa7c5f126010ecf22d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/15b365d0d4b2dfaf93b184eba04aeab5d37a7767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/9bb2bc74f6bede7d326447c543d249accfb29168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/a0e8c69938015ad830ff12b9ae883cdff7843c09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/c5381ec0761fcc27111f4c853c7180aea8793bbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/e847151f1cd26aec59cf7491c7e3952629ab8390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/f6447064d3fa5d7d40903265c51d1b5b48fc8bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/fe01852de096923f92abd8a7a1a450d77b8800f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/11c05d00e16e9b4680e71546983d7749f5fb604c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/18d373cd6d13cafeef9fd0a84f442da307dd2964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/497561b61ba67481585bd0002bdf8c1603a89d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/71792d04f396112763fe32c66f7b17c17148688f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/79fd0c785aadbc0e807aa5dd24eed61eb8422ca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/bd1b246e89e0eea6f899ecdc2dbd56e8b5fde6a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/d93abb75fbd6cbade0535b3a6193c7095d8f5572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/ecb36e73706d77d5b1626eb7b23fe53caafc22f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/065bd9c13185c409b9e291caf34c9560a26ddc05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/0db254370e78b5f09700be0eab147e7237ebfac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/168982588cd27936db4459176a119ea5dd9e2467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/3b78b2df6df95c56b9ba950af67528b48ef40e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/47c5a8bf841b1bded15b62e52ac9e561c69687db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/58f1c043e409ea5bec93cb49ca26c7e866ce3ed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/797cb4abbebcd5291d973623b6291cdd261f8b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/e02123b2eae1d8a294e82a5b770d7bd9a4dc2858.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/2630b9d81d6d4603a4ead57392770e648c2da01d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/3297e6f369b18a537547fbda1a274dd438d3094d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/4a2a00aba6d42ae01c91f5399f8c855f32326b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/584f05876cdb1546baa450785e8f959da34dfe4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/811a1ad31ab08406448cf21ded60716b187cfd47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/87861b870f102afa2e26b01177faf1fe39222d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/aef063877507e11ed13f2ca698f925be96ff91d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/fa4f55c486c8c2c9b80d768f2b87b8099d8f0819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/083fc7344460b561b658acb4fb4fcbe870d7abdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/44c1bbf678a25e4904ab30553b3639bf24023178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/46f1fbf3707b261274f620c5416256103bba2f78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/6b04baf632f12cc1a86e9a7450870643475a155b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/725545ce1424279754673984d3794a3d7ffae28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/8837d06d8623cfabcb84ed7be057aa573f312dff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/ab0bb650c3c1f0a3842628b34f7b40626db41c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/da2f1f02091272e9a8915f2596d0a5f905c8ceac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/2173e0def5e22202c31d715d0f0ea8e99506b31c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/3221e5cc7b6363088ccbddff61e3dbed9bfa11eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/484c57bb989a2a7c65c375de2073f3889560fe15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/4d0202dba8e83704e670ac7732dd2d98357dec9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/62be5567f336d315ad1a37cccddc921e4301ca72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/6f1581d53707002d66d7d7b4f884c44b22f74384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/df457459b1a51789865ec11d2b0aef5a3b00cdba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/dff8bccb3f9a9a1c3ce6ca9d8829f0a3cd836bf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/0753a62e4e897a73fc54f099f4fe92b119a2a07a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/42c5fe9f5206b864c85a663f93eb37ffb3668f30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/46de07566dbe5b978ac1ede543ea315f015fa6e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/4857ed64d1af673f75ef240538f5c9e936af8dc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/5d2cca9c28c040ab49dad6eb56170a075fad5077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/b85def898e2e7f3ee3b93ea279646b5de2885d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/ef97f7a626ba58f4cb82e51df0c6d377279af469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/f55c7116321bc57ad492e2018cea5bb47db07133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/039c26740abaede5e539fb5dbce37c9d46256944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/2396002fcdc96c0805d4de42e0203c1eb54a3f16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/80b4521d20a7cda2ce87c4687c5ec69e1a27ada6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/856779da73b23820f2c3bf96947f5732fe112ea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/a7f4f83fde30eb5ad8ddd9d0decac44d1afdfd78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/bf0c17ef3f3eaa21a49b1186604bb160dd43017b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/c0e54f1c60d4951e1386feaef3e8c6e671fbd23c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/cf29ab58e6506b764ed6c1c89e741305c9e09f2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/090d41938a927276ec69682580b0ef889f7394a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/a9ae1c5d3c8d61d72832f7b922883ec1d1a203f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/b95bd75b1da88bcd99daf9173bcfb226bc9cd1cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/c07462b8e5712ca4348efadd52aa2756629aa04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/c499399e910d3d1bc7391fc44eac67c02994e94c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/dedeaf2b9ee1a9e37217dbf079ab94cba3a60d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/eb9deaacd446e696f1e4ecfc6c2aa3d79d34a117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/ecb5a99f4f0f0698b87094c797121c25e2bdbf93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/271a91ad1d6c5136b1b1d10d0f544f73a351af69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/666ee69ff9867aebaca03a9da9991ebc8926ebf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/80a7a43b03074b84c07bb1980e3924ac433ccb69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/b8a59a0b70a31c4df0baaa2c83c054ebffbfa707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/bee7eb958e2bbad522fef70fe45cc517f512fbc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/c953753839e12492a621b3339241b2a13a906bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/eb5623db8ac6db84c3dba6bd3aafc9cfc1ab7d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/f184ced7579b2a8b9f4bd432ce3ad708b9288f2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/0399c1dd5bdc69a2825f2bd51b584b45ae27c9d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/9ec7d97b205cbe22d5e11996049e6c90f31d46cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/a09592f9d2f93124e761e3208b33b435833843f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/bf9e19ffd646c3308511d3c08aa6f999861e5c17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/ce2f5aea9f1e5f3019a98cff4249a81aca18d4d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/d3e8d9b069a5301aa970b58cded2ab4188ecd21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/d4ec5bbfbd83db616686b71fffe8a252048f1d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/d4f05ca11b575b9ea7f23a33dceefb1124e40530.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/0fb521e769d9c3e868608cad6f5d97e7e710f6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/11cadc2610f04d19663b524b5f186d2e446b787b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/248855cebfd74671194b386503cabac64d99ef86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/37e3520333e38809af0c80c0701c39a5261c1c0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/7160e93c1b559d4004935ef22a3009985871296c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/cff0a20180ede3c90b348ceb13dc63bc2de6b672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/e459f5031884ca4884faa84a20d489759ef17a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/fa428feb82cc3a8c65aeed907091d0a79bc147f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/05e03c29acfeecfd56f47e19baacae081cced962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/06a2dc4c5b9de0bf1b1c418c84859dede3cb0bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/0726ed81cd6dd85df2a2ec8b060c981000602c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/65f0be0063902bbc64a558b7af5d251aa250d948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/72341a6faffa14d847b4476b6ee42cc770c16f96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/c14e8d09d106b265f69f325019de51fcbedf1ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/dbf57a64a3ee43b272e569536bb4f21649bd01ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/e4ee094033fdd57fbe6f91cd763238a67f65d57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/0b4e4bc9faa05b73241e054280c7b5665531d49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/12de6ab9b8d2cf373979724b71a3987d477d282f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/3934bee74abbbaba050d7d648f626efe5f07e799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/6473db96225e954579fe34fbe915c9c96408605d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/90c7eed62ea60fd69f51272cdd425079809a421c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/94ef975c9fa0b17c1cc180933901b95047da0856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/b391e676a7d976e63e22faa5f14ec079b21b425a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/d0ab4bea087dd0118de92e70a7a5f1850422401b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/6673fe36f746890be98a4f4f3d3bdd389d074908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/8bc616f8514ebe9bda9526719efa511e0becc6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/90c9aba17e55b118f3a573cf2b15f8e37f701292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/97895e8a6f920705f50d731337cda64497f2f0b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/a9c55c704a70e91e81b1a76cfa1dc86265290480.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/bb405b1d0a5d580d0dbabfae518fac40349d1da2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/ca9360a463a4a6088a54b8247215ba3432446354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/eee53c8ce8508f8f87ded86ea4a6f4fe3f8b7b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/10a7df311c1087f51cb1b9b064fb06d6c3c28157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/14e63e54ba4f6ede0d6c566dbc77dfb6f9b6ce6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/193c4703fdc0728d3c513f74e8c69a7934a90e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/1b814e6968d053ace09d423553bbeda140eefcf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/73a3767c3619e80531ce0bd86070463ebaec67e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/df2819df814f9364633927e294314d85e54a8fcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/e094b96e2e30e082b8f6756972781b6672ab6791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/fcd6db8e5c8d54e71b9f137a5c7f1b0ddbb0c782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/294cec619c036a19c736351c4a411d8ebd56a32b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/3ebfd39034168c4dc8ea8643e5615d72cbcca986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/a404f8afc91f99e41c098ff3c7300d7aa9403b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/b817b370922a75c7e1593e5a53d41c0087ceddfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/b978bcf403e11f880888a302262d335c30393da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/ce58907390eb2064233ac5fa3a3e4fcac281b04f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/de550d41eafe21796fc9fb5a62536ffc836c4984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/ebf74e2ad5cfb9a74b6be222b1081934f3d48f92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/09759a11734d2f5d6c4c1e7f9d6db559d18a9cae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/124040eb9fe272b19e1a818d327c1ad9e6341ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/18b7389037ddcf68d59eab56216a3e3c0177eea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/2ddd6297ba9c2385e1b5b1c24770ed5130c9185f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/9e3ab51a358cf7c84d4eaffe3be5e576b0a88823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/ac9ef3bd2fadac4c10e59ef774144250eec410cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/af463afbadf6f742a9d9a61c9e900a4724e812de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/e0960e657e511cb7e7417038faa3235b5a3c64d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/65278705a734214a1d3f5f266bbd96e0a0ee86e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/6ff133aad07f5e614898c3cb4c3f5556da0e550d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/714b695a54655694fd767912d89dc157938846ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/84564bb779a65c09d6eafb9073ac984b96f5cc64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/975774e1cfa4a3f719cc7e456085610edb6bae88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/cc5f2498cddbccbd7cde37d5b88f29072c624a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/db28fc94a492ab40826c8346075cdbc82c4a0522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/fcd91a58fd5a928f502cc71394f0300021473061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/030b1d0707627e996325a28b2789e23e649068dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/0dbf4c394c5745dafbf686fd647b13174b89bad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/2ad9a12a4c3decead8bca6e16632286779aa906a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/4d43eb57313a3c9a950064873399c6259e9705ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/5b024565b552b2e47a027e280041124bc47e637a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/7a61997287eb6e9f2646a9f6a23dee8fb35c9141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/7cfe0e22d1b15cd6c80fea24c5b69a64ee2d6aba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/bf7b079c72dd50deb1ea7f0002b8676c41312617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/111b01bc8113fcb8572eea9f48f122a4cfb74581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/64d05656d2667e0d9e8b26147bea66d650ee9e14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/77909a0c2bff690643eb019eb7e7bd8558e5cb5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/8f2f24896dabd92cc2341c3ecd559ce7375c4a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/c57011f0466f86361562ae2fe1764ea711528d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/d683d661326b086af4b009561fdd7812653f1e52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/d8b107774e58578ed9d51b38fc8f5b1747d0fd11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/ddfab4c241dc184cf4651ea3fbe36f7409d4d243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/06af657cd60f96842714313e95b1c535c4e4dc7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/495d105cfb63e45d3a069d14a8a79f0d25f7a69c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/56cefdc01aabf87b0eeccacc6ff0ee3062620f65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/57239fc9d06f54ae60f20dba2e89c3805da0d483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/996e2620c40b11c87b8a5757157b646bf0f42dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/b7a8a48d7a1a4bc65e29b81cc8879e7dd284cc8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/dd236eeae5ba4d3e0f3b80ee7f4fc7d0adbf304f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/e342236d9d65925c348b0e72565d9ec2e010f3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/04a1e39e16d1ec0adf63b5b23fd07507c1302e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/2ff0cca44e2f2fd090b3e7bcd29c805d0490653c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/57ce505aaa35560b22794be47669e0df0a5a840f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/5e402869e610e03ea02ebbf70bb01f0fcff25797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/7779377686dc0472f74f57a4eaa4efb4a1f45eb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/88094f12ab5720dd451213ab9b18888cbb7a0957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/ae9668caafbc1978045f0d06fc60cd4943c01d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/c90cfbe1266cba33ee2683f3a6d0249e3745a957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/4e1c4d2e7052f6a2aeaf7bedced3da588b09ec3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/540e55bd421bc07b97c43c469c711f1fc7b34b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/892302ae6c9ee541b541ba53ed20fe08b56fe956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/a939c5ac631a76e32e380cc3e60755fbaa7b6d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/b61185f4f8036aab49b968a453dec4bf6810f10a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/db803c3a6acd787f1734e71a2d3ae6d778772e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/e3adbf2ee3c21c0ccd5f3754946e7b9f90b2b801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/ed191562cfc4812584194e73178d0f617b5e7654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/3c89fa6e6d166638251de1e4cae00fb22e14036c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/4488d44eea3c7eac510fcef620ee2c4357830af9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/4a2b13b0c6569089d9ea7c451bc661f7cab50f91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/4c43fc8be2e9adc1caa040ad7307043809ace8e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/7746ba7618c23321e9a25b870d75dbea4c356b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/7cb096d78aa921200e70d871ccac27773c68f9d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/8970efb32215d01651d87f3a55b503e976f3d77d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/9f489a55144243c232bd826328ddb3d27e84bdfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/50adde9fc38087aa80cd5e24a37c67c6261d1a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/762e5b3ba6ab07fa661330229316dcb91466b63e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/9f23ef839e53b92c666f39674d44c89f24925edf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/a38aa4c459228cb114810a4faa693dee15f6baa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/b66ffa8383b8b6e73609cff9fcef3816e2c3e403.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/c4817e6c27831a633e877ff9ac94dd315ccf22e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/e75f07c9f447f4659824e002ae4e62902c5a6b77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/f7d21a2f1a839344648225539f14b34fad27dd52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/055f6a18332d59e55c7510214e3e5c37339dbc03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/2eaa23a9e7b9f21f22a3e1e342bfa4b3205f508e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/70642d8cee3c84cf02adee98b6fa2f8dbf67032b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/775996ac61513ef2a9cc28283ad9cc43c6755b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/b8b6b7fd15845f8a096042a996e782da73fd560f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/d75d070ff5de65c375ade3a5641140296793f9f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/db87924e472bc59ed74e8101b465325e7acc90fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/de36192b1f7f6bcf19fd96782e9168d49c1efdcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/2512b6b4bb82e44522ac46bf92aea0ea52b08f46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/3cff729c82d4b8f0aa16c821941a2c43c8dea036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/439fe83fc682552f612c398bd2762364c0da36a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/724f6922753fdb282e571b7a88e7e2d6cf0ae1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/870a2284fa8dfa72b6f2151e0d7fbc5e6923609d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/ae59acab1f1956802cf6274c8a49429ce1b33b02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/d60942ed3c8b206b56fd31ea955d54686aabcfab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/e312b1a53ece5a6441e155d1a82f3561917e9315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/1392437cce4d61c460512b1af8f2f718288a642d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/39f279c14d546b65ded86f4ae4a17e9a692b034c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/3c923a56478d74e843492240f7e6d8d919efb2d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/3df790b1d3b82fe8022a6df66fa2340745299c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/5b9f5e2da84bad5adb7e6481cf29f0826023a513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/e00263d02797202523032ff002f083f0a70e09b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/e61067f533a5c058864356feb47fdef474b314d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/f989d309a41f8514716e4f8957861f17295ba481.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/1f38fab8dbd2dd2fa536b9079b8b36359c98248c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/3c012082cec921e8882e4d0c6665e17b3242a3ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/5260c9b00c0fa88db60c48de8af19645c94c4657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/5720e10cdb65e73fe2fd2434b9eecad8da7b444e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/6314d584ce80b9250971b4cbee2f117c5b2c621d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/80cd73a37eed0de02e0af6c53c9300047711a732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/912487b8e8c41bb00998b2797fc598ba3a44c145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/ed167149180b752873dad22b7bfd600c5efe8d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/0c396040d508199c7b23046982bf9132f7728eda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/370090169aeff87464d16de44843e9cad306571e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/3f8520bfcf1009fb3b4276a8953759c3250124f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/55b4fe2efe7033a99d7fcc94ebdd306962aab292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/69214c9fd47aa8b3941d876d9f3e11507517b4a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/876d79b4b452a79dd73203717d834596c950a949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/b9c5e7e27f9277737decbeeb6d4b2f01324ff5e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/f5b8bd56e75d7edf523a45a998feb4bb2825ba67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/435fb6803512ec01cf6d8f5a7487f64c188def8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/44e14f1b6d74c43004bd4b41649e0b80c31a5785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/4f1a9cff216f6a4a8b32a1a53862421ceda7a045.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/70ad84e5801eddd1f404af48ea0763b15678fe78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/853004f4a9f58cf3273f32e007689af6fda62259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/893d668d0d6984d8cbcf372d53968134f4293f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/a15c9318245e51368cd0284aef69df7706702e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/fe54a83e99fb30faf881b8e8db4996955d3515dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/41bb2481e94fe51b24edefa06e67242c5b46fdd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/62e218e87de851e87d79c86ed98a9f7ab90518db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/6e25cc470c593a2b3af7d68594d75d80fe3189ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/772620c5b493322df7dcba4ced081d5d777517ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/789c11faaf6ca2bad9319eff591dd970e5e63814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/a69d0ed0b785e93cd5faedebfccd7de1d74f53d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/c5484fcb6e8a9fc5583eaa09e2f229171def85d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/e370988efe07e082c11991b88b1a7bc32f7b3332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/1672abb0d636ca0a5441737e075e79506ee74c99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/21740e9051832e6ce71060319623dec74aea304c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/3dfb92dbba40d02b15e99274769d3e4d895d1c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/42d732f60716928821bae58b3bef019cfda9864e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/63184a591b087de5425f2a92961e7fd02fa3db7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/6e70a0c719189d54ff9dbc91b304972ae30a11f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/890d5987f6373090702e4da7a42a50ab2d5739de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/cdaeab50029be43bbb091f694d77f1c88d95ea19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/02b57abcfd7a3b341e693613d5d2a9699d28ae23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/9c3be72b9e5f5c3b88e710a1fac4bf3d72577ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/9f76679e5051fb087d7ea0ed893e02dd41df8f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/a393e589c07fcd63005068ad9f67ceeb9fa91afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/b855e7b2bae2a8f883e12b9f427575a48294d6f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/e69198d8d19f69917fe63d8dd32e0599e3b8264b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/f41151c0b91497be6b711536df1620b64345dd5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/f7b7e110473e19417165089b4da4b3d755c0e7ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/22b7bbc489456691e88def31c16f0b6f20251a4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/2dfc52c650707bb62f3448618da3eea504e324cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/5b324a7ca7f9f33f048340b81416cb89066990cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/5fa7a22d9082e0b979e1f66d1d32acae5c2000ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/73698ed838517f136b03594f4c8d37e888497f4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/8bed93bdc5d684fb34d0c45d888f1fc6f78e6f65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/9d24e95e30f493a1f457f29176f86c7b6493b3b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/b05ef518c446f5044bdf8acfc9fcc9aa530b31d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/0ac4464797f5e4ebb9b121eaed570db33cc0afe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/2bee71e35ca12545fc3cdaa1175c0886dff8f011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/30079a8e7f3bfec5d12424b2024a9930af9e3976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/317d2330ded202932889c254779c3b030833fa0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/45b884bd844110ac933c76e237432cd5f5baa591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/60c8fb637c3ceea01416083ab33550a84406c97e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/8febf0f44306bd843977f664e4ff8e1e2498d3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/f8c9471fb860dce5812df04a50a8c2700919acd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/08c1cddba807aa1bea4b9d9cefe2135a285a18e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/22bdb7f5252b6142ed09dfe683485c200320fb6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/3ecb73ceeb7f011bfd89233c41076fbfeadf7b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/6c9ba5bf4a3ee5965b00426accbb96607d5abffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/a82f332534a949dfe0739352f21d06087060e1d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/e2c26b953ef7d1672ba40367d4008290733c7c75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/ef1dc40e67b11b87d8d80fb9f17aabd462ce8c51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/f185eaba7a6d0ff85c153470903fcba4c8569711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/0002e62b24b629e82fccd41064d20192fb72fa72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/305d8cd7c924f7bb2c4587b5919bee3dc1b1cd95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/615934b994432a1d260666e9224e498889bdd725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/8b703dd9a47840ec002c88b0286f81de448b485e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/a56233e6727d8723fc6f803435732e65184f60fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/b9cbdd316fbb2f609ddcf4172084f6064b693c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/ce24919582a512f4a118c2e72795c1af7e313130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/ed016d7c8bfd735cdb8ce1728d22b5e1ab1a1742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/09c9452a6dcecc85df667f55c8e1ef2fae73fa50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/73304884bc81459d6a8489a8a822c75b28752fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/9428fc766ee4d684860e661aa2bf1a2a912f651f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/a652b038e3847aacaaadc6f9adce754735d14d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/cf84457dfe0a34142e42e99e087eaa8ef1c5b29b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/d024e88bb8b0f5c73da25599bc870f2c445aa95d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/e3ee77cb18ef12c1e9ffa8f65d0deb897f57565c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/e95fc7a7340bdaf9afd0838756f3789a2ad5cba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/0b67b6fa4b2a3a8be995bcb852027d491e40854c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/42029810843cba6d29b212156b239b4b0caeb127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/592ed810d1125905cd2a5a642f494173ad314df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/7aa3c324c7d0d43c5e6bd0c1b70707fd387e89ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/8d578a3bd95103cb4d41a82c92892f9531f6e3cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/d0ab5419b99c6caa9be3b07457027fa1b1e56f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/e55348253b86b7128683eabd342c3340c0a14583.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/f2f4f2644ea9c62ebcc59b818bd6c107550e8153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/2d925e0134bf14de834e8ccea1e3928de982ff71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/3ffff291fa8d80155e10688fc9ae2dfce28ef84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/521956ed2f2985e9c71f6728e5f98d1fab7ae7cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/932be5ffa0228db6be4d4e40758d7c619f74852a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/9858d48bd65a85a0147f548309ddda995db30d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/b7bacafdaf8aa807ba05ae00cbbb2d6c8e8a93ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/dd37a81e0ddc98ead533127af2dbc890dfcfd665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/e2ff34244bbc4e50191c1bd0a45d15893693960c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/0c204e762a91f6d2ef47a369e3aa90376d186777.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/10f84c9a3036ae372334ec2023fc7890393c804c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/7ca6cb5ccf4d76f9f5b745ace86a1ce0e433d184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/7e49413c77078db8cd1ae973344bde8545bd40e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/b3266796eecccf8959e420fff2b956080a6afcc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/d7de317038f41c955146c95c3b4c93141eb90958.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/da854284770d4657a13099f684f5515d32631c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/dae5eef4738601dc42c88030db58202993e6d03b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/09d55eda6f29ee25b6f9cf12add1f0d6aa086d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/523205b30c297381bffa296265a19b9de29f3a39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/ab0423aec48655aa8d1b5b75d40c897f415677f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/c7f00a34b9cfd1873bf15a75de27c45a32dd5c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/dcecf9d114b0a3fbde29e0f68ab650e9e77a6b0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/eeb0837b9439d01f78772701fb1f6120fe3b8722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/f0e3477002926676fb92424bacec9f4e232b8168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/fd1e1071e644bf178b3d24f6e8940581490cf5b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/08f392ba953addb5bf58b8014a39c3ef877046d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/1a29bd989fac150333f6f14b7542bd5190dbd264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/46636113566e5db92927cabd749033fa5965c48a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/8d9e5db2295ee446865612767acd796d273fe6b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/aeb0d97990ecdad9dc64b7c26110589933606489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/d19c3a3833e9744ca5865b0d218f21fe589dd945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/ef9e5c8f543b0941f7c670716fc43caf5d69a090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/f92fdfcc8c9dff33fc23c3bd457ca98b7dcffe4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/08ac5e1742abcf6bcfa812ce13f3630a8f757758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/476286902928acaa037adf20914fa53e2a5b496b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/5c3811395483494e03564d3f9a3997873e772552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/6844357b13e1b6f44cd11ec4c86c62825f91a3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/6f9fa59dcd60b7c962313c826db67da1f413328e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/783240b2046a72f97bfd158647ca41ee741aaea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/940c54dfc709991214c8a298fbffe006233552d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/b17e9c23e562bf70ee82590701b6eca10d7e99e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/06e279985a85a04f5a3fe29de6af868536b008bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/0cea78282fab4afabb34421a5602a829ae46a104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/4897f7d86b54cbab46872e36e7cfbc30310ddaf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/5cfe6a890d06467f7a4811645b31227f02ab66d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/80bf0baa08483d25fe230a583dc208e9d094c2b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/a242ac8ee4702106f994965c0296d5623a6fc35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/b133292ced96838abcbcd7fda260d513d331bb5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/ceac018955d1e02c5f5f172f20b23c612af2f096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/1aba50d980b49288721f1b7e81762c5be34d9594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/26b8cbf0af0b889626d3d05b5153004d4374acd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/37a6f306ae5956847035b634da685837fdf63c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/80e6ab78ebad46b677c1555070569e932b6ea28d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/8d71524ae54e5d98feaa71cb7346ed48014d102b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/bd62b63d523de7305f24b3b29ab74e0ec1ca1b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/d4a101ce1d9e87477b760e65d531140bc758e4b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/e311c76e2da5820c4ab3c6202e0dc9074fc26d68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/12bd29eaeedd0f4acf80884f04a2761d72e4b4bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/459fe36dec9eb9ad1d9822d8f6227ec67b6ccf72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/74d5f450a32c207eefa24d2ea22224a3be86ee76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/8a61c17d173af70694f5a7ae191c9cba6f620900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/8d3f0d76edb1d594703208ccf880cc34e356a536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/cbeba177cc3a7409d1dd3e5528b46547faac783a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/e5ba83a39ccc5a3027b345552fbd52e116b8cdd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/e96fd8850127dec20fe095a20e2fdf7a5761328b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/0d546f31e9bb4c1ac4c7053813b1050b597d3ac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/121ece4792a7698433f79eae7086a4243a5a865d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/1f49076f682f0b348b9bdb6b3af188c8533290f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/23c6a9976e65da06c6310e7647778248775af419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/35e2a81783ebed0af9ec004795b744ab3e094a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/4beccec38953f43d111d68ca8e5378d4913b14f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/92db8fde76b33be2e6d128e6fb2e42b9b8bd2204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/c071ae8b2851bb033e6728983fb157b139b59567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/239fcfc8bf3161a3f9899771f0d275d2ef088e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/617c3cc43b3bc23585820d49a20428988eaf1828.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/7c4334d7816e14fbdf070e106ba5a5aaea0034e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/83b1bad6dc871251aae39b3a71fee5dfc3b0ff39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/878444fbb982b70b91199a6e26c9838e6ba5893b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/92989835db466781694e2d0979215d149b00661a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/ac5ebfc33f6dae026e86ada9ca7ab0b26a55ab2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/de5d514f0f78646a6d7f5cdfd48e52c957696786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/088819bc3692749c6fa314bc4c2b56bc88b13f53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/16a190382367fe37f5a875ad4e9edd823ea9ad1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/1ded20f61fb69585f667960cd6f54bc908d83b7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/63ff6f41b774f00feb2954dbbe3bc8a56940ee94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/9a1eb756e70f2a6799b7a3f87e36b098cb508f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/9e46cd7ffc33bd572ab4d116a184b654618e13ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/bd88bb2f694199ac5a2cf9d9b94add37d4690be0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/d8379b2805fa0eed78293190ce1c6ae28ce0f14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/3001f2996a5c79e3bc291a1c98f8ae219fa45c0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/3015a377de32b73cab456301af965ef98224af7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/438a0bbf55816068f3d5453cbde184b747a69546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/438d40a8416d68eec4e6c77b59ae268bc65e6d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/c3997b0b20c215f9282e51970b9ffe59a3e53d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/d02072ff78defab2d27b1db147a5a6ca43f496cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/f1c1307b69d4b9be5b2b1a51e48dea546a844d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/ff86a59ac1387c685d8b87c146b1d8ed469c71bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/236b5ba43d5e6819d1581214b3d8c18848b1b3d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/243aa905ad60e7e197af1cdb4f390a1ad7b4e6de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/32201d181d28af7db62c27f38ec8c605012621fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/635bffb5c0759eb8370f36d90053c1d2445015ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/9d3afc91f12ce8db383a64f2767d2985575bf439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/ddda9bd97c4806af05fd8520a7c1c72c88bcb3b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/e013bf0597ef7a02be8c8ff1e283f5a08d01d1ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/f0d976d8794e49dcb962177a940bcf0756adaa42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/4e329abd4f6797f852b7ed095c77747ffe683e0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/70eae92ea416e8e0876000f08a414d87219d2689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/9d969e4d325fea8d70dffc65b699a18a2e37ff7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/a17246760bc49543df29dcf5cc4968e319d80305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/b1e3cfb6f4f39f9368ee987eb60b5cf364ec720f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/d0f29c946e9838367ad13a8e9a8d90501623409b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/dc863a6e3fba2d3d46b7770f16389c6e28cd8fe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/e17a03645184eb4437795b5119c1669cc7cfb549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/277b0ade3dafdb5be29e085e8f69a23266965a2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/41e8dc2997a9ba88afdb26f80d8bc69e9ae07e26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/4bee612d2914dbe0a7fbe27ef55cb4212afe35c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/5826d7d8d00e62158e6b8a9dcf89796fa91ca3e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/8bbcb0fb3ca380d4ec99112d3296d84554ade071.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/8de83eed78add8462da72c0d718282bb39be9248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/948b9db5fc4394c07cdd746866df90061ba2f931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/caf1888d503a5e8f7fcc1e2141f4e9c0b8a26242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/1ab3de232a0f82203f6cd3496fa712f832427478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/1ffac2e4a1ee66839f904e08c2b9194a8246d2f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/538864812a6d40302b94d030df6e2b528c6493d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/7e43a45dd48b6d8c1d4586c85c50087c24a98990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/996d2f64b9bceef5f69c6f07555fc684d57123ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/c21243015aa1bd97e5a7d4e2397079df96bb9c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/e3a5bd9a0f0f0de6dd785b60104f129d20da0dad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/f363325597eab9c49444226c8728fd4307ae762e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/324331b44622988448d68a37d08c943f2dfff28d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/6b843310b796fd233cf196047ff79cc54c7ccb2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/6fb9f81649668059c27a4a2ce1f60fa348469ad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/82d2586481a38c386d6f218760a81c41d36c59c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/91476fdce1a43a21e7415cdb22994c4681004587.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/9a3d39e6257ac2655f987fd7d07690f8b27786ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/9ec2f70f912047d0e3f14af161d52ad0d77ed2d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/b645ce5c008535262362ee3317c42884589ddb40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/118d9ea0cad20eb0c9a238e68897c1df2cde4879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/18c0722eef2487a3e07abe03e40b2c4fc586c15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/3a7fd8df3722cb2faede01847a104d91eee541c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/658b1aefd5b41a18b36282e3c4ee8475f03ec197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/8a83d9d071e7970c52d9a0b2fb6f8ab155fed67f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/c468e287e00255c771379c2d45c9dbe6609217f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/e60441c525e8315d4f6643e9f3e0a4ffb7c3f5c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/f16221d990ee659522ae774dbe61bc978affcc43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/0bbc828147a3786050e61e6ddea17df451fd0648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/29d18ce91ca3f1214872f704609003172a6aea17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/6cdf1abc0133a14c201ef76175ba48ff0fb6e113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/9db23f75cc8ac5438e943ed1e83e77d4e691e18e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/b78699c0ae7970af207340911dc5739b8614d4f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/dff4cc0d20679342482f79c973b80acfb2b06af1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/e475ddb3f828874c121d9dd304b7afad3bb2949c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/f9b0a5766e1cef07415a30f90b5af3f479567265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/1519f4e619fec79c6c4daa901702d902b709fca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/3109808c6f8d99007ab6cf14bc3f61df526c290e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/336ee97c7433b35c143fbd3605bf25d9b0f413a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/3daf78580b56b3af24265c6d9d2581eda591526a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/41839b5398f4b08300a5e8c78a213d4a3ab01ca7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/468d7a58e6ab32984cbf2c0ac6399491dcd298fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/70261dd469308a10b85387e1b0f4c9a636aa67a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/c47f468eaba3041186d8903af4787008d9cbc72f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/0b29d985dd8c94ac3bfeb2d0a400820caa6dbe4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/196a44e1720b8b8e7874d5b49633437f354a0bf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/278bda3d02c86a42c517189601b42965977cb4ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/4b4e07aeb053ab00b4dbb0e65ddb7799036927dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/5cb1280035ee87c7789ebe36a790288b3a877853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/930e972990b1ad85c9443f2e0ff3f5afa24a3a38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/9394da0214322181fdd5501876de0a59c28a2c16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/f2937c30e962bce732a165ea68cee73475f8aacc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/5c3d5a91196acce2f2221d7dd8c1bc5ba817ba6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/88ecf1c648de8fda433e537f64d9589408f9fcd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/a3201af610717d6811b9ca79629792a49696210f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/ab93c8b1ef8b5a49c57ac449547290bc41a81e41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/b3a79a053786e6e9ac04b84ce9ca0e5b567ba046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/c47bdd88d805b656262a2e04e8bcd3f41fe28bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/f880a7e8dc88fa390bfc32fdccc2bcee76e82f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/fd112b03aee1f53ee4698af5c6f7a6a380c19030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/4f4652ca8a77ed645cfdffbf9501f4111f077a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/7990086c4e07bd24393ef1d4ded0e6bd8961c582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/840e3a95d2c35258e229f0da6425e8ff8c14bceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/8a9c567c4da87f65b13ca600315035de41f5dcaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/960c98ee009c8f897212590a3cf3d9f47685631a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/b8ffa83e172680d4ee59367e3ef42fa65cc07bda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/c8500833c0f4efa123dafe92217df87e8e088cff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/cd8a1cc9f49ff04a9f23d0da768aeb5606ed228b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/3a9bd5ed49f0748c278d60c1e1ef4048c9b62ec6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/5920b384404162c512ffa8ab5ea24069817293eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/6de7fcbcc53b003f7fa3553a16aadd6521fdf2c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/717b7f04962e51f14d946ee64105597c1cbf4596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/8610da4338a2ceebbadbed31b8e308a0ce8590a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/956af4bfca620c8b197ff16298a6370d6feabee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/eae45c13a5b33f06e64507c99fea6ff5fb484957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/f7cd0a28ef1332b086c046ff7947555049357717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/2bf55cecd8264787914c5e4b4377b3749dc0bde7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/5baed48684a1f86cbaaf4d6e9c3fce66f37a4dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/6d8ec7294b039b9b13caa0866ec915acb13a81ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/856f46710c30133041bbee836741accd44e3d9c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/8b1d7436615c017d3702b38085185efbe0080531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/8f96568847f31bc6e5c42b9cc92bec32dbc219de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/b9112c83563857919e7c4e61f9010632878f6431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/ed15f6ca2ca399a40fd15c7f786355359c78cb99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/002787df7f8dac1931bfc66c9948faa4ae751aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/1b53e24599bf7d78a81c224c28de40b9dd30e075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/1ce8da47ab601d4db016639302230d47a55324b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/4e847824f40b041451368305a081a4a9b240fdd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/91639e53af2eb02da01e31a96e958228999edc91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/986020cd7425bb4adabacc5809e5a1b5326cdbc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/da851f377ec495f6ebadfa2f5149e7f105ed51f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/eb12091739e079083021ddd0b12c7ff9961e0f13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/11feb369a24a5b9b9284225235c3c83274334205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/2093065d9ac58ad5805f427b10b25d32f6e9538e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/2a5279afe026ccd79e62b3bc267293cde0e4830e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/3939f407d53b68d163734a6de723d683d93f4940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/62f0e562b8132fb588edb91d46597d3eebe41d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/9462b14dc2861889406d292979b441415e6a447f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/bf773cf6ee9c9eb000ac9283418add52d361f92d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/f5947362ea3b5cc3e56fd62ff77f75423039ebbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/00cdd7245e3a6cf54555e7b4062a0b7a2f378324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/03ab0d3579db336b338a285bfe83450c07725e64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/19f769840e5ad4143948ef621e03fa3cb8787cab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/24d39b86b57d8729bec719aefc192679134f4a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/412197559f7af86b4938303d724483293276cea6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/5166321b7537b21379760371feba6c49708e1693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/dee7b1faccb5455f8364372eca4cae80fdcd945c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/f875bbc7ac4308692ab2a93d0f2c419d4d0405e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/070b6efb3e3ac70c1a3fb18965f26ac35571e2dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/51c46273f14c3064dee818926e0d363f7c76cd24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/6571dfc55ec6a72fec517d6775414e0b0562db12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/6f3693962b8bc60bd23e9921c01f640794e3edbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/79e345cead9dfde3f84cad823a936f2f6d1da67c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/88e621876764471effda5b1e770132b96feb5ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/9449b032bdad1fed34e74d95536d23cd0045403a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/bff1a59e5745f0bbe6671b05d331720f886bf55d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/008a35d7eee41c73d21cf2a532480c5653d2a58d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/0304c79a39e1c429412da6c06d3de5178418ada7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/0e7e7f573e2a984846413adf85ad1a5dd355ecb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/3dfc87519be7eb120d3e370b614b0d48d02e5dcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/485a43a514b133b22f9a356e2d8d22033f28cb04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/500831bffdf87901eee19e7cbec309fc569d36cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/7b09561f45efb6ef43d734219ef2ad9ca41a5697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/9ee930be3c2cdbd6b2ddc0afe5d6b0d655690687.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2023-14/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2023-14/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2023-14/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2023-23.yaml b/configs/cccc/dedupe-url/CC-MAIN-2023-23.yaml new file mode 100644 index 00000000..4c9c06ad --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2023-23.yaml @@ -0,0 +1,819 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/010c15ab3255be92969cefdf9460a9d9af48c9b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/1e8f17b4bce4edaa78f0572b4c3c006139815944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/2404f9d7e5e6d5d5bf51e7fa917046c6d269ec42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/496525dd4a50ac8e5944f90bc9bffad1ecdd0baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/5de9dd9663e488a268022a340d6c26964db70571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/664f5c9c98f11e146aaa5ee3b9618bf7d447c2f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/9074ece84136505ce45cf9582be35f3cd2998613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/d1ae557cb3e2c4664bc2e5a97e8971e59c628387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/03c40bf8c4f60d502bb91456888cfdc6f6acee41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/310a421a5ee5db08ee2d7c3bcb6b43d03f67263a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/34567919a736a2ce37d700d1c54d1189ad7a57a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/669d6be52dbae0a14754e25e906633b7631fe259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/6db48b2116827ad996fd1a919dd4cbd7de8b6fb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/9254905b584091cf5432936dd693ef9bb53731a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/cf43c2c5550792f0e3e52fe0528653cedb2f2031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/ec875c305f5a83d1942db1505b51465ac093bfc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/1d5580eac4052ceb2b81f980ef5ccd84e720e769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/3598048ec09eabd6601f3ebca859507252fb6a07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/3973e90d5145ea15b69a21e0c42a2230beb3fe3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/7ac1c5b3ebc435ab9dd3c07b3947b6efd96c9f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/98888c8c104d238e682394ee504a7bcbda0cb1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/a9aebec496005f8abb9204f46301d889e573eb00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/ee6204a5a29f1ba519fc597ef5bf6ec19293a406.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/f3e4b11723436bf640b5b871a4db48a43289355e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/059e9d53e6fb9454376bd09da7d6539cdb95b1b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/503fe9d56b9204c2ccb6362fcfbef0a5e682a866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/6425f96619cec367ebde9852985c817b27935638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/71aa624574a1043031690a8b7eb36b5ef52a98ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/90f6bda744ad4c7bc6cf3777890288b6c8be2a8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/c14872abd92126f1bb28962db5ef0ff0294eb3e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/c328a5c0a18ddd17092c134f416a4b33eb0762f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/d87a726e39d0efd4139ae0086abd1e07c04ba303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/249b8b422fae9a2933acdda4886dd41ec2757e98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/4b9440c34149e65ae36371e6747c0076b82a8894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/76e2da41d3435287ec163200b7fbebaf292af827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/911a0c75c733a323156532e96894ff6db35d8494.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/93a5cbe041f10a4e75902609b9d0aaf25a070394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/94ae565cfab0bad91ed3b31f0f8c5a6c5f654180.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/a5185ade388895eab09eb20601995f7a99748d5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/bce43399688555e5bedd194571be5303f954d597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/02fed207b44e2f1f3f71b3ccfb0331fdac93fa1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/13ea15d0c48c1226d4d85943b01a24180df80a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/251e4f47c22fd11d757fa4f99725b2c24113c30b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/2dc66f3e184b74f90431fddcd2cd2386cdeacbcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/88295bb22cf8c941f47d87c9ab16210c05b2ff63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/afce220517ace275bd4f21633bad20cb7c3e2cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/dd077e1e375b929cedb8e848730a67ff70de5b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/fee80c73b06fe9312b9e92729ab58db25cac7f5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/2b058aa0b6ebd3b663868bf9a3014c7316e1d3c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/3a613f225c4830a025209b01c7b5aad01c0a6005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/471f232a3b650d00525724d5d728b0ff467eae3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/77c27971ecd15ff570169342c6a537f4e7a6517c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/a030f7c483091949eee42b306b2450d185bc8228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/b2f5db48b2df5df4173b61e4788d6c7598d7904a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/c125122a5433ce4df268803d57638dc7c357af0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/c48ee6eaf3407436247bab7d67acc29ca32b7995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/2635295769896bb20a68e012a6e0c3f2f1672405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/2a4824110cd2a2256a28dcf901081d18cfface9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/37bc1704ba39d5f9d885b457d457f9f8466551c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/3f9066d367d52b5a8cc563a4bbe8a3c371a709f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/6553c2ab0d50b4aa83b2f0abd81d4a61ff02a555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/ade0df96e54e279559bbbe24ef052950571fb6db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/c731d08279287abe05a4b16be0bee669c5c05cb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/e34315f9e7c0ea13386c6ac767eb1e709160e89f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/0bba12b1dfcadf43f2c14d3e0db7134587222bb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/49a152cb1cbaf867af916eb3046c6f9e04869213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/5b776afcd0f3609d8e849d089dc292eeb26b2337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/5f944ecaf0a2b09e719da2d85dd08f53daf03163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/8b514896b7e049897eef4069eeeaa364fe518a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/bd42a4923e440fa1b6a57869855e50a806837b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/e9ec5d914001a777775cd0199b60b8989be8a566.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/fb4721b6b1075c25aea5d68f265e99ac8d41a2cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/13dc3b3744a13c9bdb937b6ce470b764ea779d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/1ffd254d7cc4830160c15d620f1e3950678a4963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/47084f851b32ab5c3a3d727f24d05e4767f0edb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/4eef3f1d4049a3ea9b7737be909f02a5feb526d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/6a71ec6975b203b6ec02a224e8d9cabaf947d5bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/7a2c28c7539df490c0511aa9b0c8c7fe13c65f01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/b6df65c01f3b8b3a18f5e8f045ff2b9c040ffbc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/f7bb9244b1d17b655211c1d94e786faebb53545c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/2081f85d7c73fe068b3ba336b8a249012d5667d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/21e08f64f81e83a333f5eb0a225406c29204b96d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/2c61f77bcd63bea1466da73409fcde65cfdb6494.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/32fcca939da9464a4845a571a2bcca5e51764b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/48f2688d950b9bd59e85936271bf0018d9045d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/4ef8df9a1748fe873151fb0097ca58646906ee02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/825317a3f47f65487db00440e801c176cd5d82a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/a50c0b6b0468762000130049587db5f13c01b664.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/038631007db7c6f76dfa17849d5ec5a67a52a62d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/279f748c880f5c2b30ea87e8cf758acc005fbcba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/5fe9ec2707b5b6db97944fb92b71b9ef909cb234.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/68a0e3989c87368e3d56a064379f7b9e024239f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/7e5ecc08fb80255de550b0bb5e00a46cb8726bf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/9e5b949d34597262d0db0b77f9670b452ccc43ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/d934e49b5c0634c8e9fe8a3dd381254bcf577331.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/e9477274dad1b1fb266a4e0af2709fb2c1a86814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/1067af6bb6c36567aea67d91665d09a98a012a3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/2f3417bab1b18565ffd77147690e7944440f350b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/77b4973cc06aa050acd486480c24cc92b9ecb55e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/7b401757558339d65630026e90933e910451f41b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/87db8b7dc89dc6e358a21e906e9090913ec89f9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/a0c87ab5e9d00c9e6026be41caec382337217d92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/f523c01abeb2475a5d6aa02d68615a0751cdfe0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/f867ebdeea6a46344c03b0ee3d235fe09aa55916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/0d9ff7bbb54ebdaf9f5e43e336b35791c3e78a04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/47741779bc28e4f1423bb9cdad12b8f2c7cec74e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/59b8a6b763e5dab4fc35dee62f99d01e44b1d663.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/600d282e543c8ceaba126f9c1276ca856fc8141c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/880953e24fc0947f37b7ec47b9ba64dde42ec364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/d1ddb049415886a47b29f03548192df87cd4fb01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/d2304e51b6a938444dffc4c7223fc858e7fe9202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/e666c1477bb098af55ecb00dcba22a3e564d340b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/3f4e8d0a8b4a30c8a7aa960dfa2866cd4f09043f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/756c6c34709411187d433f12f1289c691fdd897a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/8e89570b67400b1b4e90e8e903a800407ef34f4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/904abb5a9d7ffa1f0260a965275a21480b47d73a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/bdeeacb3d6fdedec76c190f20e552cc340ead315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/c1b62ede77f4898d4d1a172d09beab1466a465dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/dd5df98669ec2aa72004c4a7421c1dd1055daa5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/ebc3fc99a8ffcd974fa4a789c11665155d46e951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/089685be4cc9cdc285c3627c8d5eb131e692a9f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/2b28aebfdfc46bc1fec3db95992be3fae688d514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/77e93b21857ba387aab23d7fca70cf43147e4496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/80710d478c8abe24086e854d1bb920741deaf190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/92cdf504097953f189c00a54c0ebe2b7d47d2bfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/c5688aa7ac1a0d57a39d436c4a77aa3fafc1b0e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/da0e9ce8fad1490c0872a9f878bc3c3fecb0dc39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/f22e055e2e89038bc5415efbf3c02f7b2402128f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/49532db4943b1526dd636dfff32c77c787593d78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/59a7813ea375a63bf8dc5aba3539cdc5d3d62061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/5be8e3b09e3d4466349aa26377a78315858f8f97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/6c8f33ca7ea3c17ff5bd15c306d10142d73ca09a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/8ab83e2ebe915f9f7359d11dbc3a5cab9ed25428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/9ece29e6d65c6a7207e757602b534ba9eec0e2bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/eae85754255249f417606170ef388b7dcbf8eb16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/ebb3c5348a03d4b3b39c3c3adf89685517e10d35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/11e7af1c224b97fa02fc72ab76b3209010136ed8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/6c26fdc8013c22d22ad7b0e790cc7a5f1c6e78e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/757442c67ab7d0e48236ee5a3b204cf8781859ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/7987623f4fe5f941fa623ecadc3fc9c1c597717d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/81f8309f40e73d9cc61192ff52bd5967415616f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/a0533aecf69d634f41da63a9e9b906b5d33dd53e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/ba33711382a29b73ce741d49b473c84d7025b721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/e42c46b9867341e80b8ceb6f997935e1bddb1ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/2965d34ea6c9c45f3d890817574e9c2d679bbc7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/7c8deaae80ff0105a6fca4d0f9844db0b002ab0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/7d0a11701d4c408d227c320840f9ce45523021ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/83a3e0a93c49062eebc8bb6c3b82112734def37a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/8f5cf184d3093924718c7ea132e4e9d3df852827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/cf40cf456191114223fd60effeb68f7dd35282fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/e1826b29350753ecc0d0a5870abeb0e56df756b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/fa6a4af7c73ea0b215736b5afbb780c3740534f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/180777742c14ad559898fe20f72760c68601fe02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/32961ce568cda44cfb34e1cff207c251d8c3ee18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/495d12b3a73f507d0443460c1a9c7ba7f5e907f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/4a0250adb51a3c13e70981cc2617749edb565948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/652ed0c5a8fefb0ad3aef9a4bd851211cde97872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/6d02830fdae9bf82033b0e95aaa90a29039ed577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/c1070b52d796681c7a5d9d81fe10994d823d1e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/e7914843ddc048104c28e179df50e750c7ff0cf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/07d41a6cad8941aa29b0fc1565b781b306f58ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/28e826a6c69b58194ac50f26ae11a6c21ee89050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/5c869dc6c19bc38e9d4f4f3fc8455a0afe536a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/945cfb9ffc9c155594e9917017769cedaf1002c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/96c5726c4c2d1b7da6ce1ebda72d0695218d6d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/9e400acfd30f6731ed7c849b9aa1c79df20a933e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/a7840437022c24b60115b90a7c7216f2a3d427f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/e1a946165258fc842875dfc99b0e515edc39f8f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/2c14f992d00f6589eb393cb7c510dea6861a255e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/356b8610b0264960d0bac76d3579a41cb203778b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/91ae409c510778c275dd92ce80338524c39f6b24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/95773832057bb860fee76f713165ab2cbb3c15e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/babb407ec35ce97ab1eaa5a1a6a6856bac91d6f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/e1b1b8f3427b680fa536f6c1946c6e9441ca1342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/e8ee79517522a4a20775a41e05622bc0e994089e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/f05751c8056c14c039b18f18628831dee8693adb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/0d3a7c9337caad1c0f97c33721a5b5d5d63b742a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/1e674ebc452cd57b166704bdeb4755c84e9c8f93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/2476a8c865fcd2b248ae800621948ba9a688537d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/4f702b9d9e950ce0ce7e8bf7a96ea2ab4a1eb9d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/639d65f8fa62ee937596f3ca662fc8b328ebf30f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/e73245831783afd8b331fbe9821aa1bad62ef697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/f0d7d34b6754209b7d56393aa3fd79e2e4afe44a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/ff82b55fac47539b0c3f3237dcc766508df8648d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/048820b5d11d6fcda33bcaeab603952977d377e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/1795fcf86f4e5ff9dad65065a86f41b799060296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/3148de32baca75a26d6c681d4e9f183277964756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/6a8631008e9f811734e154bf0846dd241d2a7886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/aa2e8d952449230236282a0968590a4382e781aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/d5618e9fadec42624bd609a58ff253dc14dcb057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/01283f9cdd33183864d3eea4a370550568421568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/4bf9c87212f7a87d36add49ea6f2ac6b3228617a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/5890787953b93d9a08e594519866b6f5c3346391.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/5effd9811260d4c3cd6d98ee6bacb3f015b25aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/9dd14b4aa1f67516d04ae113b7b0dba24c21a551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/c304a077e2ef3a63b83ea54de8fab379d7911d62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/e50d46a12a9ce43331a6a64be82a6d0b8da5f4d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/f0a05f923a78b8091e5335c208627b525587040a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/107089f83e969068fe227f8000d46d523a2deb03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/3006f7fc50aa3db5e46b3b0513c832224da33613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/3218c957347a625123a7656d656360c6c155facb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/597c78d0e73ab89efc9c46165f986851a9c8e048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/81456213ca885d702b2ed41362cb29c455730e19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/d978f1e0f2e8d2ecce5baafa16fe1c95eed47fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/f0402fb349aed87fd250c8ad18f14ce95bb381a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/f5a6547909a0fbc07007e6520d14672984831d2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/04ac2a937aee1ab31f0fb785f140839422545fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/1da06fe5b891efc2c2cfbe3936a1b5bb01c2bbb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/360f035decfe8996248f6c22e8188f60c8a53c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/36dfa708357589ca66c7e18801837bcc96625c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/4b4a8f054b1228ab234414fb634570017f3d1cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/7a9a37a5c7ca22811a3cf82c239b99afc0a5ae72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/a35c52e34679280288ea7e2084d07390c01226f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/beb6ebe5d1e7886f0b6c8fde41881ae732995e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/0ad70885aaaee822f03759854023a0ef61070cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/3670420e78dfe59cce0eb1231a869baaf3db1666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/623f8ad6c295eba3700e9f190188ced7c9f6f577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/80d0a24b2a8aa2f7d17ce06884d434b29043c8d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/8c61ad0e106d8d0bbfeabda77c806968b41b28c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/91e38461f823aaaa37c2d73a6f035acec7d4bddd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/c2d040140c290b8618ae671693a64740476d2706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/def84ae011c0cc63aa1458359beb5330e928e4d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/2080401345f37c30d21287aa637d88ccb3c2595e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/2e480f2a53e41876df4b31b2fddd0456ecd2eab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/6a4212db8b7f89a912fba3c8a466964b83e8341d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/8a221ef42dc82b41e8c5b64682f71c1b93201855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/9811e664206b39287510fe8fc125753bbfe83398.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/d370e318ee688609bc151fd0572ed9a4ee124ca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/df56d71fea70b9c2a900fdee498efc4a6516cd46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/f046444f808e9630a18541bc01f8d822c1422121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/1a5e9e2e412206dee71273016217d94574d9840b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/3724bec1c7c71db809a34c6835a1c034aca769f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/3fd29f3cf0ee3cca6abf71e5aacf0b1fbfeb1b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/600a4cc5e550a95bdf6438293b3494bd7410c649.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/61f6ed7678d57528e970ccec18af60c336674a92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/66cdc4ed73eed3f1384ff08d30212a21c46abf9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/980b1486bbb523e0b6aa840f96206e16151a4378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/ba67777d031f37fa90dfa41b38b88177ff07dba1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/06b0b00967f81e1f3182a5f7dcc1e2c65fce3d56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/125d8752082e59253f7d36cd30362d179543819d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/31c35171c886e6de252df9c439d6f8fdd6608330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/49c2d3fd610597fa514bdf88ae37fb0da20ad395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/63d0836ddf34af4ec087c616665c103b9054fbd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/6a2d751a51d9e0c1729e5724837bbb463e545022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/8b0c182b2de7462771c870b2f0cb0c02506df90e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/8f36ce51875283778d7d33d353cb0357312cd5b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/377a207e786d354e12f1bd92e67d2b438684bd7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/45312a0b405a790403515a0d1dccdb3fa0f9f39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/4f5f26fbf60631d70af3fa5959f314ad6bd6980c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/5323827c4e383c2eec6afa62e21eaf796aa34144.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/7a492df5060997428510b3d6999882240a2de9b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/7e10442097c90f094042cb1beef529c9e75ee15d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/ad9fecbdd0b21547b7628a5f194d8395dabf6027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/b35868607311012d35a29c183e3f0505ea493198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/38fd4fb7777d9e494608b9bb7605d6888e6a27ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/6a97a855e2f2d709a0a5db0f861bb79294616813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/70eee31e43efe40c76ee50e378e8f1f2562486ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/73a3d4a0e4dbc9c0b0932a36e6b6325b519f54a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/822d1cc349ebf0d968e3add7bb4299f861f1785c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/9f27fbf8e1f486546398ba424e9b08c48d2b4f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/ab979d9a883c8e86b3bc0a55765a76d82462dccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/f54da55fbc8ef2a4ed3f2fbd808c50d437ad7a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/125770ab1ca285ed6d021a67b5c284fc722f1ee5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/7a8b45effe240badda85a8c3ccef7471a2ec4e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/7f1d61e6d54526cf82c42582d6488b9d690da773.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/8b9274b83169d8e8a588c4c97332e235159d8050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/8d5873481ba30b4ab614b00e03e0ebfd34326e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/d682063897c921ba66c2a2817201f0ca9a7f2490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/f95513fb0ebea571d12830f4c6abbb6aba24e884.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/fd512184576d021c25014b5aba00f6f1c961845e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/061abc07c703a8bdcdad3422972dea67cd334d8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/1ecb8b7d4d5ac4e80199c8bbbcab815fea120380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/21e7316a4c2047ce303eea5ab0a10568fb47d9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/3176c712929dedcf9f76758ca4c7ce9140f87d2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/93056de2e992737f2a75495df2986bc6d979336c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/b5d7fe9423495fec14b3b36ef21443d330149080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/f0da6d2c16f4c8544d9d63b20d1271209d109368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/f86e3f2738b2d954d0fdec4efc2f3f79f1e617b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/6636cc26c88f60447fddb3b8d3205551af4905d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/7e9acd4dee2ea9dadc816e8ec9ebd90c6a2df83b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/80e5be7d6c364228ee11086c832f9066d4ad1229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/b184402f367c2c1941c0af866d573d3171968cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/cb68f152a6fc1e58a3db793933c3804778233f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/ede8cdfab0d9aa77a05cf8470ea254d8b148c13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/ee2464df770cb4490cf16e582582d3ccb4e7826c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/ee45367cc8022ec2c4c379a39df75b3743b732f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/0eb28f5ab7f633a7f02d7b4e547302a084224da7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/0f4cf219b5a5d059176867a19b5da1ab7011b6d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/4575741ac07f03e82a10f69f672c01d42419a9be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/7a5774c282feb4bf83c864e345838f491ed71fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/7f570c251a93d3d7a71b1b3a126fe374e2168c35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/bd35c1bc6e01efab567ca59a920c1a177bd36da9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/ce6a0e6d823f78265f94f41729a5a5d2707d7b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/e8d07e87ab2d3e4e1fc362fd81b40438418faa16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/236e0d1ff9abaafa99ef69f14a44ce996bf7aed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/27c1e8ff0c268f36b0faa36e9ea53235e289a78f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/3b56e779838d394a29be6694ede4b712a8581269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/61e024742f7ae56890d70f3ed3f27bf5dff32a36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/9ab7bc31d2aba2c4cc9d61ed0d3e3da9404cf4a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/cbe193aeb4605f792463796664edb0d95278253e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/e398ef5e5e5112f5c66ffa8e660ad9ebcefccb48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/e5a1884f59b3d62910d7b88202bf211cce256558.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/1f23886806f01900cfb6f4765a807b898a6efff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/2bfd4b4210beed0e8f513ca834bf1fb1911b1e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/37c167eca93192b1c2c60acf31acd2d366e367a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/4aa6cc0b8557be2182bacc861f6b07cc7d6a71d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/5780ca59c5d1bdb8a41ddca3f5d72b620a4627cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/a65ec8750866a283f7fc8d9624d6052b80368038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/b22c3555a0b980adeb0145497888c081b4dee37e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/b77fc342a7a0f9dec65cc2b46c25d674b3e41d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/56bd7710701b95e34dc05fe6602c1576ec869524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/6338f9cf4779ad5725bde7d89054309a2da186e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/812288f93341ec4f6fc2f75d8fe18e6d565732d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/a2e197467073ec583807a205279e3c39d2cfe553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/ae91bcced83aa9b9450daffa904552ebac27d8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/b1b3b8e511e296f57f2b4c2031ba63a69bfe1c36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/c282d4954dfc441822e94abfaf7ade4b2c9351b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/d6b02b205af3bbf0168146a2b9c5f858ff8338b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/10ee6363e5b71389289781b018004388933c7b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/210d3735b9e974008c4cbdfd47bc2d8d272c94de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/293635937caeec36f0598d35079abf43d9062de3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/527b80dbcdf1cbc5f8ce29d40b020a277f42cbd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/a86399c9c8983407a7e19d4ee39b20f22ac63d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/c6614ccaeb07156231e55491363440f979ce6278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/cc0ffa2edf1dca36c0344444063149957db7fe98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/ffbf23158fb5b90f61330af7fbda9133ec4429bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/03d07a3bf989b40fa34dd00b00548505e29ce8df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/0cb1c0d2b57b7aea7211fffdb02f4c0cfff4d2c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/1067093166a18e5db1da10ec59c902867c321f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/50df87e45523f829fb1b7d9398278f8413a932ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/7612cc53326106c2d2553436931dfe3898130bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/89127ef777818e93c86ebb4ffc0756e425ecf0be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/b3934f5e04480559d69ff22cde56edd49219da1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/b7fa7c39484b1cf88d98a88df1a86f6c15c062b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/3923e62b1b411a8bfcc238406e58c159152bc829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/770ef47f6bb4eb4057cef82e80f907454c3f1aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/80ed39cbea6ca6a60acb5454b83feb4c580e7bf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/81273421108201205a19b39239cea00ab6a6fc0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/a04f5a3579c9ed015f570fe6f25bf0d75ce792f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/a0dd44ff4648d667228e0b4089277a38ea56aef3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/ab3ed02c4772d7fcf9b1c9ebe01a4013c3bef54c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/de9c5e0cfad93c62affb7729b44e6664bf2e3758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/256b8260c23721adcb050d664cfb6e5f99568091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/701bdef43527ce271c4d3f5722d9f2ff18dafd22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/757abd83d3c896fa47fbc5b0e80b696d20535341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/c3a36c594a3a8e7de0d87c3344404b96e9fc7038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/c78c19e8c47ac09f05976d531fb1473c85fbc463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/d9109af4820f63e61ea63804d8f08e1fde3d141b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/db401786cec040e1a8db353032c0312b8638c703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/e227c1ff2151143c7740256c401ef6c7912eec99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/30b07c36d037147a090dbb2e434d43ecf8cff73e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/30bef0737edd7464e28e0b0da39212eb44979fd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/53a2104eaf9463230313534862218da956788a0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/9dc354b4238ff039960a8270cd1ead2803547e0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/acc7eccc94a136b589161d7219b68c80f5b3abfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/d97b5b03ed1a59c7c93c860cf323f4712bcf00e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/dfca762fcaf915a548664d8aa0062d4e4255b0df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/f9936a720e48ee4052901e166bfec5badb86c372.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/1ef44f063caa709e2c277ac52b1a628d6d34cbfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/2322d4275199a30c4c8477a1197c4eadd3deba30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/26240e7c6ad033df881e336cd09f3c1028f7f732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/3e36112d8ae29c662b1e443027401c04888eea45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/6c79ff174de81a4f1bcd731d98738a5c5794c118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/84c7f9aea8f6decb6d08323b24bc8c02bb89563a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/bcd0c272324f61e61f5c92de397aa53e6f31f12a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/da857bfd53346599b979b149da404e30b8ab4d1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/0414ec38f7a343b7ce8077133b33fd2177ce706b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/0d6ef0ea4b95fddb34e4057a39f907a66c26534f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/63e70382a2408597ccfc28b7ac8d0cde79e49a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/6ebb1b6d0d43d87fa041bf83c69d79f7fdf8be64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/98eb1343b5fc6c72f272cd53a8ad800e161017e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/a44f5bf04fa436303961463168e58d7ff3dc5827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/bfd0e54250fc30d6b0bfbc976ce4cd2603a2c23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/f14f5657ce8a2a6f7ae7c13a72bf622a9435450b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/1ceb766f74f7fa50d27af8d0a33e2752721c4db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/25dff178ed97bfdf3d627188dedf4d700d8c652a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/3767d6b3d31bac0e9befa57f52532bad5fa5fe2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/7fba3fa7da82014116a9e9301a0fcc67b43a7e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/b8a6e68deb9d852a1ac6e6dca6ddc2cbeb7cc696.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/bbc1e221ebe2fd9f30439866700daf9b1d872d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/ff220079af2aa6f0c13d9c2e9a5c7ca73131c8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/ff9b5f9f6baad9fdacfe6e75ce626e2d4045c07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/3e24c6ee2faeba58eaaf9c1ab673340acfc7c5da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/3eb118ea6e72fa6ee0d84d71fc2f4c81de413350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/4e9b62b2a486345cadc864c75566476fe0a4e005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/66743d0e8f03d4f637a85157fc795af6f7b6cbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/aa030da5e2220d63fb91a4a79afee61f4bdc4fa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/c403a22b5f9ff962ae03bb7663a89c3c3b0e7fc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/e2a2dbb18317190dbc157975274c831a93ad3957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/f7bd679213e094b5c7441a2ff1fa16ad17305557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/5522b75b9e8fd44f3f79d251db4bc97ac2c5e853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/9001c5a972c67a1d487ee30347fc1311fa803c13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/94693aa6da31e04ec3872d9ccbbb3857483239da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/bda2980a35e0d18b227d80e596da5ce94315b1c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/c07c1ced00750f258eb13dcbcb629346b13c8cf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/c17a4d09bbf470eaebe72679deda19886a5b8f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/f4298fea9b165f3843f313fae6f3c71c7e50e47b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/fd3020764fa507a89db82faf2e16fc6e558f0929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/0364c291cb3bdb5854ef23a165fd7fba3cfd03e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/17dda878026352275f88b6396b80be2a2a8269fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/256d8113e8087b4e4d4794fa0c8385964d05bf2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/27e9bb161e3dac172ffe1cfbc9b1cff43decb602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/626ae4a4b9a8981dc8c1680e782b00b2b9d73ae6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/8326601b5bcf45f5190c9a2b5eb7cc2ef242f833.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/90b63cf7505172887d7504676344e9f47f78fae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/a51c98786c681a5794628f8ebfce9d3618b2fc54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/08927d69daf6766db9fa17624aa2b207df1c66e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/251b2d9c97ac370c445ef42c8f264d3619afd58c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/4185a92c5086d4c2db56c1ebe07cbe7f04f21279.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/781c76f9be630a9966266d6f4f932b0490776542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/a67e51781039de62de39cca53f7b175cd66f5350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/a6feb5a945015a512a6be3677be0ac7d370f349c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/c3a33ae8f678b5bd9b9300481056cacbe5327783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/e8c18dc022b2a3cae7efe66ce9705db7d4039daa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/149331b8810b7ad9cb7f91469c16a88a7e4ded0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/185fa11a6dab6c82bb6bb692dc367a441d3ebe06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/43faa262adacb1c5b162ff19087572a422b0c1b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/78c167c68ba373d3ad2038a7d64fc2729ca76a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/7d948edc1b4046f6a96f1c9428e87d856fee5a1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/88ffeab2f2b6779ba0655f356e49fd8838e22a18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/9c3a13c6bddc547ff7c7c2d1f8d32ac728c87980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/c0c47be6c19e4b5ee4379e2c3c4d1ff772815b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/2886d98ff67ae7b94cbf9086e3621b71da48dacf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/29579a2eaeab2757bbcca8cf6b875e8143a3a56d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/312e69815c2a6183e0c8c33d37d3a79928a3af1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/3a94bc3fa4ca24fa819fc516f0833038b393cd2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/3d965cea902c325c1d056c71e7b30c321e4b44e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/6d49b91907df9349f50d4807d9ec2d1be075fb44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/aef7e65e7a31c29f60892f3a6370db141a614ca5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/b3dbaf755f4f6ff2672d201405b602cb02957274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/03ff429783d8b08e6594cd79fbd734eee1f1a71d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/0b8aa48c88b417baabfc7645f77ff72ef6fc9b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/30c72e436c5cf38447dcdd38c90f719bb4c0f9b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/69772264a7382131d3d1a6ec2eee1fa9c99e151f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/ad4e8aa551b562aa6ceb79b9df6d8e56f0904a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/c6dc8e6e1a41907050fe17226daac7e69ef7f0b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/c96d18abe71fabbb17c0ee6b35017b565bfa1cb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/e3471d4cccaaede2b515576f3d9053f68ef25934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/069ccd027a2afc613308bc078ab558087a06f8f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/0c31cce1afa6f2ad5fead84966daca98c5b18740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/168b5744a6c559a65cc7d24a7e80ed50e8905aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/1ab5679ca033466c32abded19f359f5ebe72fae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/44ea4c79d27ae7a781544e347331e076d2aa1cae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/4c5c16b58dd11bcf511486558a2a6ee2c445f060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/6ccbdc8721919978f062223b616fcfdaabeda65e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/a22fcaead968ca38b28e577c5e0246818e23b5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/3684b65cab2939801aa7155b283e8396334df048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/4ab428a9ed4425dc946a582b2f8632fd0c7a9cab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/4da71768764fe693e316b28d47e831ec08c3a550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/576c788d66407a6e12997c7e833eaaf175dae6e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/59104b6c99e15786ecfd30ae242c52995462fc45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/95f8c888e14429953807364b67fba1b7675c4189.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/c02a44094ac9d416b3574d441af1e4cee0d0cf31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/f23b6ba5d58e344f272339558fefca008308b7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/104a9848588b3caa8f25df0409831a6a3340f163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/1a84ee62218d442d922dd5ed220e93dc8da6567d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/3a2b0dbf2d47f89e651dd6bd8a01151e1839b509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/4efe11936ce8e773a7b90841063ab133abea60a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/6431584fe6c76f7ad0120ee5068001a5f43f6445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/65347457fcb621199ec01110a871abfbe1b8dfca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/84eda4509b62708ef34d146275168c90b70bf9fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/ec3e1b9bedaad0b081d4b1a1ed2a6d333aa83f5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/272a50667b4499f1d247782449795c92f73387cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/3d0f5cbf614ad115f8421604a874e8adaa724ac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/4d08d37d2b34dbbdb40569b673029688579e06a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/70e83b07621c2bc33c12c07815df258dea348a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/832d31cce507e253d8bcdc6405ea1e9a3b0d396a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/9eba48c9f07fd7eb677954d19849160a9f8267b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/9f165b90f38fb66aa069497c8437fb8a98350555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/e1ef7015788350904b544741d664cb35e796ba94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/0ba332dfd636fa3a782b11b4fd53f91e46dc1f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/263e6b04c4c6f3f10583fdf5f397cade28bf9a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/3655524757baba10095032af9813f5b20d315bb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/46df8081dfbf72450abcc3036d9ba4a00fc7551b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/698a58067e723059041a168d4854a004506d01b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/8915eaf0b75bc43e650028995c3004d32eeb0805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/a18f5d707c52c6a1c8b38cd97fd06c70ac6bdca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/ab221f731c27d8caf59a8d720e3221d1fec29e38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/37ac23313b791134590f3603fca9e4d3a98b366c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/5ad21d3a3e0db1d9308d61628e8b956c6e4cd4f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/799c83aa5d13b7a6afc027cf59a175f107956bd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/8bfb31458334e4f73ea505f1223340367e55811c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/924ab4c709179249df75caf8b52bf778d92ce984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/9a3c7dab659a56ebd14cfc478aff975f32a29358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/b3bb5758ab86117e9d7b1bc6c1ea3b1b417f8cfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/cce88a1f33ef06f6fda80cdfcbda1945cd1f5c19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/1b3761aa000936341fc534399691453fcc9ec291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/2e9ddec7ff71f63079cc80ed889337eb5e9c9dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/44264802e14d8e2b2f931d544d4b28b8973a4604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/5fcc74fee93e5222ed35d6de5b634d4341988dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/69382b8d0876211a31682a2a7c1e8fac19746e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/a84f96296cc27c29f9648a9315517d7f647e5c1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/bfe530556b8de425938e724e0f1fb7fc1a1b5a07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/d9c5b21bfb34a85a034c96432de8735be463307c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/1d9df5cb198b771ce190fc9260ee691ff76bde46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/31388c46e17d72ea605ddc97975bfd77b71072ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/372db6e5720ba6b335a7acb81e4a036782286d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/76bfe966e16a77ab03a93c95909b912b99df324f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/984513d5d40672394776656469a1c1132da8ba35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/a2089794d967d2b049927129b94aa2976b97255d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/c9ae721d58185cefad969bc602aa55ced6e943a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/ce2449baf4ad1abc030ace340ed81dfcc3bdb87f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/40844fb1c86981b8067baa3d5f11be0caba7f74c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/56c9b9a225067349f6c901adb5007db182d37272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/60bd119a5cec7c70533e9715db7fbf1707bc2882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/7893d8f3bd3078e245c25342d41fe0cd8d550e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/a5b105967fb0820a72503ca3e7610ada4d9cf9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/b2412b24a0e57b96f1fcf2e2e6bf96ce5fb909b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/cd6017eeb9de1a0a18ba573cd0608052e27d15eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/fa69d2d59da69a4623fdada2e45307c8ad6c5fca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/1810f19dd687f4c2088a3a04bcc8508a1e11ffc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/198946599a0a0d5e18a4caf7f696f1c031291b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/4169e4834ba69007a1195fec7af07fb5960784ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/64a52a09f05829caa72070f8ac3cf0873724d732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/723bada756a7decd6505afacc792337c5eb00b11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/9dd4a198f22fcc518b0f402a9ca089190f38c727.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/b9b27c3e3b7e32e12fe06aaeac30283569b0101f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/cc5a964638f8768ede024d9b4420fa61ea8cd531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/0dfc26b706f4368b1601aca78e13a4f93f5cd531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/24ab9b30831911cbdcb7bab2b94fa7c619c871bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/2f2d09784239291e676f2814eb047226a0356ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/35b1ad73ac8e595e1064f24171dd32a7595700bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/4235fd15f48a75048a6c9da202bea2472db548ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/6fdf6162e77e6be110572f4713bcc7bdcb49ad05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/7f2882afb91cfcf3e8bb818333cd8c722f318beb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/99d05de8ac3bb1f1d63d30d40f22a19746866be8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/247671ca992590b2ed77e1ef9ca9b46b1612262e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/29e28f4a5626b9d042d8f18b4fb571c871651388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/465493885ca73887c1c3578f4a1a55bac66ad68c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/4dcd70aa7d25e5eac244db817560a17faedfdaab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/8e006710d1066388e323b169fd1269ea5ef3e2c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/a0e1f0358831287ee0742caf27da77dee80aa714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/cf055dfc3cf48db8eb9c70c519adfc60b4b99c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/ef47187b9a20f60d03f2e27a6d53b6e64edf2659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/0ab91cc5773b8e89f5c3af9076897340abbc2396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/213c6aae521dad3745b40daa4abb26f01e0a9a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/381fd579a472b14322e32482cbe83f1694c5cb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/3e0b29ae07a1f9d6e435b2172a6ce1f4acd1ca6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/82dd963a4bc86eb1221ab30c2fb5ede412690102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/9f745dffe77576cedbf9b6e37b2a11a573ac6474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/bedb35bd575eae5a7991d1572300eb935618589a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/cb75654271539d08460a195031d5bc24805701e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/1efecb24023406662cb275ca59e9fd7dc023b67d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/20c2cfd6fbd6ce9de7ffabf2da8156a631566075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/3013a86eb64e34dd4ec89fd22dbf14c527c144dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/3ce894a2d8ff28ddd7d53b7b6bb06e6969899931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/41d200ed14cafc9f712009e96f900a22a20e9e8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/7f2e4d380bedcaf91d83cbc69720c473e77c2e2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/dbbc687de30504f118e1c989d5c0b04065f4b168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/dc90aa6aca954e2da08fa7a275e2fb46b79b3c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/1e0913b4506be520dbe38eefb8e1fe1642a47973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/22e959e864e4d5fecc371060d5b75997b928d0f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/3c4b9fc09e17d8a90a8cc779ddc6208f0ccb5357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/5845ebce6bebc274ff832de53bba83d98e12a9fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/76b4820d27ee65fd5aeb58a855aa1ce1042e1336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/884a1f370690d4396fec6c2e81db6b77e90c390a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/8b17807856cb23251908467b0caa55e2ed37112d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/c8585c2e44840ff70ed9e62e08d2151c19803db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/1010a4491ebc4c8a101926a7e8e9319eee266508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/49ebe7e0a17321b056d93ec4eadb0eb4123da178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/7999f76a1c45f8c6aa69d0b412b297e8328385bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/846be8ea3257c2fb8a3b62cdc7a83acb7c685916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/887eb317973048fe0215d94edc289dd917f4c724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/b1e5f75612f1207ec072fe47e0af324adc7ed0c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/b26318054598402337db7c059d4a66895ef76d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/e0ee18b669d39909655148733a21403a9563bdcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/15cf26cce6fcbf1810c5c686817b119f63c76549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/3cbb4768eb72a7f916e153b502fc35d3c34d9505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/583f1363abb8b3d1b17accca19687688886d9699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/8f8a30bf9e089f32e2366ec71c7727c86db4d9f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/9b37b564635d6c6d0ad1878d1c1dec9191957506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/c504338469ecfc20fca9d2efa2612f75a12a4c68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/dd48015696019387b437e4fe67a189deac8d0c88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/eaf2461883c65877ee9f6d54dc4784eb07c81254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/108fcea898d410edf0f72353b04e768d889e1f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/41b5c67a70d320ef85975ad5a85daf3682e87b2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/4a89bc788d83ffeabe36979760ca0b0f0c99df0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/5363f7e1fc2d08b694f76fad7126177df2499ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/66ca6bfcbbdac9c6b865689650a5e1f71eb3ced4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/8ccd13fbabb38010e4f7dd65433822e61618a238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/edbaffdcdfc8c2436ee4814302cb8415eee48947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/fba2890325db089509065f9bee81f7083681f442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/1ad2fa23d9afd9122b93dd73bd0a06034165911c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/3c3c8a4b845368b26f1426066fa14d850432a510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/8c75b5b475fa77b79781e4750eabd86260bf08c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/978f6422646f0e062f881d4f29626c34e0a19e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/ae916fdb5d82928fc728a6bd7aff8059937de37f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/c7b2018dc33c7b45b4519d52da10cfa6a99568b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/cdf5931e9be345915227b9d3c0111bb09eb837a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/e91629e787b7baa8aa41991d8e017e959e95edf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/06603d650bfb6265ceb37144c085776d544bf65f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/2215a2f8f69e0ea0d9443dc1e74ac856865b0241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/413aee51dac66fbeb6e5b35f8d4e3c68dd55e434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/5c4870e50b1021b57949d1f476918bd43de3a1d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/81d278e84edcaa076320620434390923b9f8fd5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/bd11fd46654b0516e7c0fe3e053399926d3774b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/bde4e306eecb1fa188aaee43705f4c91eb9ec666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/e4f76cd56601847d248fe4579147ac6778aa4498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/14144c6efaa2ca425592ce248ceb0b9d720ae4fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/1d568d50bc6b01720f33915fecd74dbd886da2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/2df1ef05ac34c4ff9deb0954401cea911bad10be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/493fbaa2581ef774e4213b00451c3dba4b087b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/a7c479ec6eabaa637439a8f6c411a7a6b3ba4541.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/ab8193f0413dbcf106f9f7595f7803da8d3306df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/cba4de8e915cd6e53c8fd85a1170cc68d1f12e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/fe23bce05d74e7254fa1defec7b77c615faa7c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/1dbc7cfca6a4ac8475e85f6d413de3fa8dda0de4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/63f712c0e6df7fc39d8d146661cf723c004011c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/6fc8e133b8d7d7bfaa0307732dfab6b53ead7e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/7c26b9faa1b374e7e8735982f49e9be92afe51db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/82b029a854411ca553a30d0f02ee4e659f6a0419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/cd6846c324646d0ddc79057aeb4ca8b73722359b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/cda53b816a6cc7b34a2f6cf004dee01345e03940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/db363d2df100362f34a3381b281b7f8fd0620e64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/1c8793ca8d7f49e985e5715f38a764dc5a34ed94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/2d9071925eb81f2f8cc1822ba758298af9f3d9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/5982c3ccfac7d76e5ee770ff5408a54aa4a2701a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/bbc9aeebb2f9b064e58090b9b07939a14a77deb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/db8148126036b1de468cd7d7387cf4cb3114619c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/e32c04dbc0fe4f5fc63478d40d55701b8447d871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/ea02870ca3dd04b689385ee6c533a9dfe89ea1b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/f5d11cb419c22e17497229542e64de4dbe8018c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/41ef6c07baf5f168697535dfc7b310b2c7ce1519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/464d9fff2ea290713d80a4c9b22ad54c275280da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/49343b1034e9d9d1f696bfa4b609e8012f629f1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/52cc51665dc9bf5380556a3287d44031c6fab2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/631fb31c3be53488a68d8fece6b3a64e28ebbbb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/6d7d93870b3b778e7ea20a4684ac51d760ff2319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/b7bb6662a99020bfa8201cb9109677d41c2578fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/d0be18d729e944543054009c9a4021619262d52e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/0b4d5a0488c5d0f1d52ffec1c3099cde4312c1d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/1466942cc767864e09847970dd465ef345976a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/547fc2ab869e0061fb3448684dc11f5861300653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/57b1f6347bf5844c6d49a06e9541d870d4e5567b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/abb10b946bd3376df1886427da78234bd72d97a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/c12b8ce83d295e7428be35f658cc9f36eb61606a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/f2efbbad6c5d30244d6eccceabf53e8ac68cd4bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/fc644149b20d499907ef3bd79b87ac8d7b2b5dfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/02cddf817e9ce22abc32fab2707d0b6c21a13241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/1254fcb1a546f4b0146bc0be342d2b9fd4f1cee0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/1dfcbd2bf28201c979eff8ba6fa3f7bb169af382.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/5ce186d4d7a4ad65a2c0dfa5f8795f9000c07ecf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/8c0141eef5744107ea416dc625c2ac8bbe7a5707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/9762065fcb66bd037af5a35017a0709d342d102a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/a6a342b87b49e28a5fcbd0a8690c030253305936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/deb0720a8c538608fbb4eb1a4826571ecb230b8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/115f5cdc1f4585c2616a6196dfe813a27410423c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/234545ed9aa3f4639d03958e1314ed60634d4672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/29a8f47b4530ad1243c89b9d96bb02d759909f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/2cfc44e31f0aeb593faf3e6a142a9c5aef76d039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/5fb79ab22e03f45500b324899ac80d79d7b70f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/7ff607185062a8e1241f537bfe32c141223bb219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/97a9838ff8dc153ca33edc7b33627e629df3a552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/9b331a6ae48b68a7faf132dd1710b221049a1b05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/038f01d65e6c750dea5ea4e239d90489e3ff4744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/2789acfc3b0544d80411f2fbac5947a805e65064.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/725e4b04daa49ae29a58bbeb05e353a2e573ebe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/986bc19ed2be6b42b64fb4be5958821c075dbe4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/b3b160a8dce35799e9bd67893d75c0ca9c3eb755.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/bb28f8255a3da6593156d06feaa9a3ff6bf1599c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/ee2408a73111898b1165aa340993ec46ff82191d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/f92bec7045af46ba542dada24382ab8f7f420d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/05fc92115d58dc3f44443b5304dd82df74a84ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/170e7336f85bd0a064cbbfa861ce9f31e566d3fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/37e3daaccc3360f9e7f75a17bfadf405143e1b7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/67842df0c4d4b83be3f31fee22084fe7d3e83c97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/7982b14b364f6ac822acb3650c0cc91f285ec3d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/7cf49e1804cb1290eb52ddf2423357ba67b76375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/8aa1b163bdcb8cb8687b372fefd2a9849c4a3df9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/928c6df47856ed2aa680340f3c058e0794b4dd2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/00285ebf323f70f6de376eddb4ec7f47c8f1124d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/124e5f8526d0e81a3893dd3a8c42a04fe2b0b0c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/16f1257ace776b707c43ae8a992fd976c08530c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/29c2faa2b1ced87bed504252bbc8864a01be244e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/581d021fa0702caa898bdaa63c02f6aa02ed7f8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/927cf9a8f85ec3e128c13f1af57214a30e869361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/d45c31153c90bb7fb5de7c433fb7cac9a425d48b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/dd9bfb619435c06e4571acf7b344ad82e45a357f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/80d04a76c34817084a96e47ff5da08d02113d833.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/9fc0bd83501d6489d23e2e049568f9ef19d305e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/ab785657a3a56c7131601531b82840492b827389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/b2096b47d437548a08eb7c91055c2e66bcd6d602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/c175bcc63f1deacf685b74dfb364c685862abadf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/c36d288ebf0c4c267c9314da958571fd22f54ff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/fb1a078eff2795603760dc6007bb2bb875b6a260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/ff30197d01f57d2fe93972732c8861aa0ceb1228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/10bb542c8ee827925eaccfd8efde42b940235a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/11f40b46d53df901cdce57ea60f4bba94ad9af79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/3ab1bd85d3b174412207aee856719b4da4c43ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/865fc92b24854e27eddc3fb467f4229c962b7b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/8909c309a4d7c15e5f5da1ba0ddbc62e0ae09326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/8fff590cfe2809d9126498b0151ec0c129848c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/aaf38ddb70b1cea5760ad271c826b28118237ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/de9b38bbfa98f597df7105766baa46a153020b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/3a9043022f2305041a6f8ea18efdfdd1e00fc13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/4dbcf9df2ae68981f1329852e42b13a60eb6c3e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/7837e6d1c1a56a9bab617c0fd67e337f4b3be4b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/97dcf4fda5efe33d33dd5d4fa47a4574ed8c0d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/a8c82faa258a465340a5fa0c946300353430a88f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/d3ec631020f83c801f5f9c6feb5f33dbfe4e93d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/d8ac4800fbde39da9b59e0c4c4ccf9c9ef33482f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/e54cf32577d977e57a9ea27480b2941974e7eed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/22cc787a99d22aa2fa994e07fa47634eabc1b9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/5f114db148f64c39da4a1d6ed0b45285943dddeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/988d9af74524acbda7febfb8344be8819f9b2a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/9b3b0fd2a7f0b35d190a11ec78f43b44ab6c5418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/a84b0156f4d057afbce1747639b0ec033211553a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/c2ab850c28b89f9700fd77bb9a340ef69233deed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/d7c307ba81a7212a15399732e409d92eded39657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/fb1442512aeeb578c9aeca366911b2ece2211f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/3c57d65dff7f802fb4d94f77e6cd3fee1b636fd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/4d581b906cfdae390318db57f47a74c0c06e97a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/4eccdc0718ced071177b4931597dec11896190c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/52b2ea22ac1db7d6b377de1e247457e80bd43128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/743512b0649ace21e8cfa30c2c92d525a822c012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/da4c5d62610e4d7dd57fa107d71addffde40f4f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/e6f7cd10b5a21f24bf68708b22e217babba344e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/fadc7d47fbc1deb3b606c4eb004b8efd4bf63a1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/05ede0d2e67f2b647fbcdccafc66ccb9bdd138e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/32c5d2e3df6bcc1bf3a0fb62a34dc6cb4e1b0252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/4f963896902dc044c687299faf279c8c283051f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/5317a8e73969b0dd6cd9ca572318ab2fa1d16851.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/572e53f5031051eee44a0c968502c6e13dfaf246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/5a41ccb449c38b43ed2be8428a4c42d3b743614c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/9a1337d662eb3c16a80b78b5d5a7a3cf0dd88112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/d7c0406f357efa7004dec58917e1323f704724df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/365f88d5b570cb0dede2c11177e63aaccfb62ae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/4a5665aeefc719939d76f6c5b521efc7339cdfb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/940a9807cafbc9b1c765db768229c62a5b35388c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/af549c09befdeb896c5339641292ff4c73223562.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/b419f5256c20074f66502829ae08bc464647baa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/b988e8d86ed30a07d706528f159ce9dbaea70167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/bcca91b4083ecf310270739671b149918c6160c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/e9e8d927d35eb101e7ca9723b0392b57deced814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/2188fca2a9816fa3cdfb83ad68f872440e4840f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/5e4f2351f073b14e389f22cf653ace4b05cc7fb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/5e9a775a99acd708ff2ba655ec4efc53dfb2b25c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/98df3c8c9a0ebdd81787f9addc45e01d8b84f64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/9abaf941a357c085e9d1d1bc85b5e81ff60bc7c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/c673b7a811c6885e5ff1a463e0d5dcdc9bf691ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/cdea08d3b703fe010064ce8f81e346adb81505e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/e814868f782c38bcc3e19e999ee850488d0f4fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/16b88a11c4a1e12b22812838a05f0fb53e9f6759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/23ec1e3d768c5e5598a752f09081f084d655467e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/31563d1952cc86973bff2a0ed12f48577f6cc3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/7a8a474ff2c516bcfa05edfe2fa59b4c5df130ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/7ab7a4db36d6b5001cbb6091ebff6e46af14d2e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/be5a41453dffb44601bc6300a50abe2fa3111776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/d174dcd4f2936c89ecda83770c38cd0f421ecef2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/dd07bca84203dc19a178cf0e627ecd2f9be5265a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/09f04ead526c64f561d1398a962fd1e6b41fb872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/118e6cc3bcb70546cc85fdbd301ed6e0d137ef55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/301b171a66b4b377150e4433ffebc38109e1f793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/754827189f684931774b509c1a70c47826ebc21b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/807321147ca2d6c75166269ef5828d6a5af75d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/882a398e69b312b741742f8a49451bdb5b1710fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/9c201c7c9c3ee62b691af5d2073512c2591f507d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/f3155c7edbd44742e412362d90eeb8e9c47f9af0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/364cdfea02105bfb485b0d49b7147810ea7aafa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/680c86e47e68e1f375cabcbd9c6a2085e71eefac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/6875c290bd5d5aafe54330a7a31d9de0cfa86ca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/af90dcb8e14b9c87c9b0b361f9923a6026dcd193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/b4f16c625a3f2e7703830438e559b965adfec86f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/d94e592edc6b782b73c62373564876ab3644594e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/f0ab1b22a7c2ab161ac64cdb686a99ca151e0fad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/faf8a8d383cfa1d975147146762522eb3eb20d89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/0e1c31a0031a2da2753e286e682a4e6c639f45cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/1cdc691ed257d9dea4b73670d746c057b710f6c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/432030e839d2d5075300a5a3b22ac3c0865425f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/535bb3e69a8417103095301902dfdc1b145f29d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/57ccd8ef37af81de9e09cd52bb4491805d29dcd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/7695c493f8a0b62addf63645b22866c98b1c9daa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/c29fc6aee18026cb3e4469ae01f7160549f05ab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/ee146ce22801948eaee6fb47794a8227116d8fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/055ad88e3eec19f4b38dfae9807d78f198e3e9d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/ae0edf3deecf74f47908c0f66f03ee304afacd99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/b5d5f94cf7ecca995f678143d5936f6fdb4eeaeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/b8a6e55e6572db0d955860a908ebfe25224ac41b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/d1969515c6941043d00e703669b3442e41c5b318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/f295c85c343d9e4be4235ea40e2ea6aac0358a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/fe1063bbb5959097754b1803f420df04692d2030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/ff76f12b50d65a44e3310bf01b14ce70b33c68ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/03c82b05e0d62b8b8a8b59ef56e38f48e117a511.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/09968dbc486713436a69af7faf29e57d7d4e7063.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/6e561b9b45781e00c2aad4c885963e1af8bc81ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/a55b510fdb1da863b38bf0a2c63d6c81a8760a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/b99c808b094b09e17b306643bf08ac2899888cf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/cb9311c31c0898bdfedd9b5c101610595c0982fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/f012f6773dc0166cb9b642af9594fee4b81b491f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/f41cbc695b5f1ab2de4696b0d2c94a55dd0c313b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/07bd4d7f3d9f244c82dda62ea501e7b0dd0e12b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/1ae15ddb26ba2f982a968a915e3a42370581f276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/1c89b5aaac0951acab10a9a7eaa5d35dbe3710f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/b304322fee93dfca9bdb83521500c728c2ae63e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/bed771bbe56d160682991185aa8ece24c3d47561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/d5e18438f1642f7483972e8b9ff0b26f2ca62106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/ddbd4aa8c52efcc03f2220e51c475a617ec18f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/f702162d6db2249df46e43f7dba1c07110e9f92b.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2023-23/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2023-23/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2023-23/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2023-50.yaml b/configs/cccc/dedupe-url/CC-MAIN-2023-50.yaml new file mode 100644 index 00000000..6a8c6b5d --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2023-50.yaml @@ -0,0 +1,921 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/0cecbe8f92b0b7a2f66894ab120a0a618ea81dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/1e7b6c1112407bcd893633999a216743975539e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/3f45f62c154b843e6cd0d2f5327a6623d2859925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/57b3031b1f4608c4ba6236ff0aa7f3124c186f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/7b3430f1e45a2c7ed0787d6644461193f9b9814d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/b97b6860f32424db8088805f8c54c3529e847138.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/e9fc88bdc1e90010c6f20b2927f3a3a9f0123e93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/eab8868532937b76b50decaece3c9b72ea6c6075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/f2213d2b8ffaac9e85aca42803fe606417ab950e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/22d1472f4d6e5885df94fc6d2d135898badfc8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/39047a64e5ebe8c8946f96cb3eb32646c0d20e03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/4cee62c76b14dcad929b3b06efa5ff736e170d72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/500cb8ec51aed4a55c11e02b4a9cc45f9d173648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/5340f10a81aebfef231c9373e9131aee0f57f028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/5eb6790a991c2834acc0667a9912ae7dbcc27417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/a50280f04ffa79ea003e581299f9edd6f4a07ced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/ee10765556b3f6d446f26b1ff54493d9bfbd53bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/f206af06227e0fdd76b636e4216594e49d178e7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/28ac08a9878370ca64a65cc3543a7554037123e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/2d01d7997ca851bcaf8977325775900ce5ad8853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/3085b1bdf6652766e9f05fb4565d9d6a2ff7ccde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/335816ee0e9240ba8f318c6775151ce2f7161635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/5782621dabcd67795f19791a747f4850d330390f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/5d630c5158e86c66dc356103be407ade3a13737a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/cfbb14cc5803a52db4b12bb5e08ab86a815e35dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/e2fe761f30b7d12e2b546b87f314709aed534561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/f722e65a8c0a26bf7809edef55f01ad724a11ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/4689a64f6abd910bfbe32a239ef72d75e699a508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/5b0b17ffd489fece2d5e0113e199bbb964aefb59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/6db6c3491f64fdcf80b14ae7d714a360f4180afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/76cd1a431cf60207cde7f1c4070a6fcb7ed0d1d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/ab8c8109be781fc2ed8f18ad32559f58f08c740c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/bd8767860caece5686c792a2a09d9cc11e75610a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/cc1af3ba2b031af1b16e40596c61c5b397d630ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/e0d7703220c909f9a084991f6317ab70a7de7eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/fdbc0701a1ef72f80bae8729107f062db28bd15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/2bd05b91993150e24fea8e7e2add3cee9bdc2d05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/3ef246eb2492868516644fff9aa354aa199b7720.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/85e3b681d88c57dd9bf61b373d6d0637f681d5cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/9f10df47f38448777f6f563cab6947853e813225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/a2ac36cafeec9d8c51039c97ace2d116dfab5393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/bf671117d09b9b5fc79c5267951ffb641abed024.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/dc86e476b3d3735734f94124b42ea342817524ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/de6bdf2eb5ddc1898b9d9d42cedb6bfa126cdbe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/fccffb44a9d9fd1292fa834795bd50924759fc3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/00afb54860eef830e59fc5eac8cbc6ee813d4905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/0e59655468e7c62ce2ec563e02b76f70ff57a1c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/256439042088dd4d43334b7cea19fda47820c870.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/48afc92dfdb4fb62eb7a7f16e19d2b45f859c8f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/8aed6d07adfe0bdedfbb5e045768b0f2580a33a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/9e183b4c5b27cd84e9b000830cd3df7b6a51a9bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/a883122994131ab0f127a4466b9cd234c204c701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/bc1cc4c23fd3b40d79053fa4a920bda5f69cce44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/eeed21e3490d1b2e65d3258421f8137c694f21be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/0ce6fa1211f623eddfd463e4c004099df8bca156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/17f00e34b7abf3a52f8901ef73c0223e71835368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/29d1367e92e8e03aa3cb20d93cee205ce81a1a40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/65ac74093340c276ca3ddc5b2debc00521547084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/76e486d925d65b4722600a8dae9135b8d27a5a71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/8432f9e31723eed2cf32de14f130101be851fd58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/8f9d31572d113ea161354628b89e3813de74ea42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/a2e86e1bf8aa6871067fa3e63eb0bec9d374dff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/e4b60fae81ff081c6bc80447aa7562ccc33b91fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/2c110b60ef4f480437f1d626d4d0c7cffc274696.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/3e06db0bf8e32bf7d3ab17cefae7fb917f2bea1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/57ba017737368507fe89c9999164abf46a1690e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/5895788e4090eaf9f7e9046de2c6a4a1c98cca01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/6d8b9380c8dfcc49317b90b7118fef9200a60dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/b3b8c5a978ca558a0d9173601e07b1dedb3b60fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/dadf047606702586c58004d455b77664a15405c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/dcfe5afca916b5108f7b0bd1c05dc592b13b5bde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/fc56ddfc8ba4f6b637de5e3a9787306c65ac0f5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/220de2e0953d1ff6b029e9087040d0fcbc44fb8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/36cd71af91c68b04af54fc65634259823a05497f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/393d10ea3779602f66875c2425139cabff500388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/583a2ba70a56d7069a2b5203cf8fb6be0492d68e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/7f3995ea410f7ccd48a7dff92cf5dbb6a3c528fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/b7a4b27da991091e2839919bdccc89f793938dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/d39a424126b64689e0727ebf3cf15658145b07a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/d7f2bad6628b920d8d0b03775484916b0257e048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/dae6f36306ef2e8b555cd185e80dfc7dd537c0ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/017419c7f2d0fb85753de1417d6752a6f74bbc0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/0da33b0d75511876a0ff075305797bab7627c3d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/2c0179c59b16a7defbc86e71f05d702777daafbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/33ef30451617c5e32e0d443e3479a5e75fc91c60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/8675cef79c549af2262232258548a5215537d31e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/c924ddc1be0636f2533ee261db9b215866665854.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/cde91789efa6eae07497bdf2685f162274dc9865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/eeb19054a4cc02066b35170bbf5082f61d7f66bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/f94bdc53f880fdfe50369bb515bedfe125e8679e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/037e6334a718935e9fbe8d87041b1101009b4681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/0f38915ab57e913aa024305512b93eb6035c1725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/12c5a444ab88506b0d1e5dffcbf88601ed93fa3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/3f7b402dc89fdc59faeaf218ebe1603b9460d301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/62b584a785ad362c29e83c8447ec1bb43866d176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/69e61e04920dded984e6469d0959003f85ff9a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/8a165327b7918cb2c45e0327e52e8ef58d0314d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/a6a245986491af729d0d400640e2634b59e95357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/cb1d560484ab4b232f9a86d401ed4c136bf26d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/1497f5eca20f87873898c16f1c1f3327afba8558.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/336b29528e7e728845ca973d5c2c8645825e8816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/370cc69879e1c449a898faa58ffea3861fb3fd8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/3a9c34197f381630a8dfcbcfc968e1659b3105a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/83d9722060e1898bf488300075b3b66162603757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/916d93fe4fe54b8f28f5e0d256e205b78e91b771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/b78d570225fb2877cfc415d8d6fbecee53949d14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/c577b6c83a5c3d5b35210b5c29f6ba964cd6d0a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/e6a8e56e0f662bab963e9332a834fb6150806d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/0ccf61350dec980c9b36d2b464e5b2ad04739735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/4e6c59ad88fda65b7aad1e33d05b1e08b4648a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/741eac6b682f774c53fddd3c3c2d72d095ae5805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/8f0bd8b076400d0ccd7d0b07ae7fc10d961dc367.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/afa00716f1d3ee99d63e678a4b8109822f13e034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/c0e0eb28974a4157a4c5f7a68fcd1476159d5909.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/e426e333bcaa8bcf781d8123380feea0444d39c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/ea2bfda629313964dd73d83c8dcc736429157dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/f4bacc5be7d31d0d7b614fa502abb988678ee142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/870b20d663af912e5ba45c9fddd30167d5496bee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/a249cc1712f200d76a4e37fc925fb6fcd50f53c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/a336b0aab8c05c0f2c839c2d1158256cae06e69b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/a6ee465df52e96ad9285f84f2a0c9130bd3e9b2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/b7faab8a6e5fd84da959d54af109b5563db15807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/d32b4921f0e3de33a9fe7847af1884e19434729e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/e5824b4a5942be6c6489395526ad578cde2bde2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/efffe615107ee480846fb6e2c19da5d0757b1c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/f02211da6d0809ed15405f4ab74b6c7531896252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/2b850331d37d23f5a0d7b2d2e810cf76520d0b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/3da7a3f77751ec1fefaead010b72dbea0243091e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/40f56c8de0ddf12286d43f115cb2d564d83c2a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/64638afa3a7abcbc2270dea339ba4ea28b64b5be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/77e2b8ea16b80f5a5bcb97a978cbd91c7ab47fc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/adf07965b8df6d6bf3a4c4788654ac5d7cea4e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/bd96b239a858997f8bfa76e374ce57c5de584845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/e50724c43537078e15e3360f103e21920d645505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/f3ddafd1719f0f8e7311cc59b52b82fa5baa489f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/0399862cc6ba61669e952277ae16d2cd1bb83ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/1bba828c8e11ee2cddf9d65f1976444b808d456b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/4761e06cdc955afbf6d269cfa8ed0ce923bf0ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/62971bc68b7a3d4874ce0e99ec59ef00d924c044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/8a88ed754ed7f3e44084be21eee4fbf7351f25ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/91f1c94ca5ceb86f8bd5bdbae764ac3325de2e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/ac4b982ddbdc4e07d25c0b3372163352e430dccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/c5efbd691dea8f477a5df8ab572ecd0e886c2470.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/e0f573220dfd8ad8fec44796e8d828767069dd7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/1c435912da1bb01a1760a4494d0b1631562064f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/254891d6244e5633c59ada64a7dbb524b35eedbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/3af46f135383faa1a641d1f13a210e4d2eca9724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/4f89e56b8d14cc660ea71b58f6bf118532bb8bb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/663e8ee7d0099dad2483cfddaca1f97b95a695ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/764caf1950ee39c57ac4f2c843a78668c1bd6901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/8b916baaf8f6fa6485ff1e84c3111ff52ca95698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/b05da1304e227703fa1c73def7f5d7419f668ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/f2fd89cac4f718c6d9e8e28a7c8a4d8503115f6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/54835d45c5b3c232e360baa9ae4c542a8822a01f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/8f8f2340dba509870538f66f1db379f6683f7c23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/a573cfc87526414f745aff7aca09a682e6fb442c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/b0009ceb86ad00179e2c54c78b68cd1162abfbe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/c55ec24840b8ec69b08618406537c03d46c157f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/c5e51eabbb62dab423549b1b5d125e8f00bfcba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/e05c9260c2572fb69d4e15512c0cb28c1bed494e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/f0fa9d20cb8754efb4d988b73d3ea87c23c62597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/f81ddf09490fc958b99d76bd6f11d99f51f87670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/0bed91879863eee5648f7365b646ef32ba9705fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/6309cafecad622a16663f9426952fc20b04e81c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/99ba782a0727a450fa237792b6c476c1a467079d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/a825eed211b4ab74c3ee311803903b450e243f5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/c0f6d2170ae8b34f951243f98ec7c49be43b1d2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/c30b4cf7847bac5d774b4d9d10bbc9e3780ba395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/de8c65bd73605eb9236d88fc479e5d5d63cacdc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/eccea2d51f77dada79c40dd2e120a98abb77a877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/fd9a2d03f00b624f9c8cf59e4d3af6eceec0d7a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/3901f91488d6f96d9c21da10c8de315e2a0e09d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/5582d0178666b811d709403ca412999f665fb079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/7714db740b95788299fbbcf60aa4a10e90c86734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/888dec5cacfb7c6de887959a30360346c171b507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/8a186c01fb0771c548349f83e95bb4c8cb25efb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/8a35badd18a21b586290f47d0571fe601ee11b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/aec6df89ac2831142692308bd0e9776417b1131b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/c03efd838fc72396d08ab97c862b8e5167dbbbf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/d14de1b7dc6c5b0d25ec923ce6d20e4ac28cc6d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/1e01de38bfeab825658fa5004f4ec8fdf0dc93a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/5664d795aae67e337d133ef1b20ac787ccd6fb69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/7f32909ef5895cdd816002ac69712c6d2fb8c112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/85de47b0b75b4f3e58b8a96f9dc645375aa5c691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/ba854a19c1c8465795745428a4bbcc8fd7ca1dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/c229f9cc72dc33b57c1516a98ab7c5f31c6fd2e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/da1546033cfc94ace20eb9958776ab1c5a9347af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/fa051eb6c9d1edb1d5efdd275f4a748f7e78b9f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/fdc4d786347376f4eb1dc49332c2b7ab625bb92e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/06e8a73a2a991d710305a11324cb5165bc8b727d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/1bd26a69cf89da5ac11ae02e152ded5e0e11d63f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/360d2f4c2f54cea2f10f7f5b6e24db7ca81a67f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/466b5fa05766374740019f07d7d1cb1f496eb618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/59555cf53a4a365a259bfcbcc2640dce9095eaed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/8542fa2943d9a2574ba3b8bb97089ea89aee1aae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/8f4c14a53a50448c5438b6e2f53d2460a6f59253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/aa1a9c92e2d70be2af5221db764306f467e27da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/e6de4a75a2ec5bffba8c7256c4fbc1a9db56ff72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/22a908b4f265d69140a9e19da39b79f23046602a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/3548f6b23abf34cc663ed85c3a60b04b9fc44531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/49bb98eb661cad7b236d2023110ae94cca677467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/57dae09e4e5ce40d689b90d6530624be2c2c4a92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/7585599f31f5440a2a8bb07af79952af25cf80a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/8403ddc21ff77528eff4464b340dfbffee38e9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/aa4495e4c6b46e6e20d7efc42fa5f12bb0b9e7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/bb221e716b768db49d4c8fc41a8ae9c4c3914e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/cc51999f2433280f3af3de08406b13289f8c06f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/2aa69a8307b4cdf8559190dde4986a2bb3a1c8a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/34a042bac84f6abb4be3e055c0811f93dbe0f66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/55083742638a33abf99531404452a429ece45924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/655262ecb4f2fc441ce98e072557bb308e9d0eb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/7ac8753b65a5803e357d817b06181587617942d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/b0ee0cc77064807392a46966d1da4ee7dce72518.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/c9e6809e648f2b8e6326d579ff2080a9a500487c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/ccb7fe56377afc9398229096927438551e477704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/e3ece367e81f39d19badee4a2a7e04c86164bd5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/1d091450b6aeb81c27dec647d9ff619de5573030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/3a0a53525dde01804cfa71b8cef78dd45da59303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/512692ee7fa01f0603a65c78214e85247ab5b8b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/6989abd47ccb708260d067494038b759143c9401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/6a8b898f08a3903ed166abfd32ac60add6a06fa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/89965854e38449644ec0fe2e1cd1bfdeb413cbb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/bcd6d1c2c4c66c2f0f3c9ea3d77d58429b993b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/c7e824f672322933d8045934c4a738bcf6e2daff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/fdd15f4bccaf758ada8238f3dbc0c20a7dd78124.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/14d4b824c3466d0290124d52d72398014417cf9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/25eb8178d2fad6a45ce0f16de4f6273f0856dc2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/4998a1ef79904643339d6b0bfbdd79b3336bf6fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/71913081457f541721ded08674774f77566b4449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/96cd67ef6762e8c67e43f2dcba142e72e5116476.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/a0a894d0d33df0ecb5a34a925de897e4b42d265a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/ae51bc716beb9b54cffb988e641732e5271de028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/c81a2f69f2040076246b4ee4e4c6e94d8c3203e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/d29ff63226a78d86d3f627d50f4d769b20006f26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/141b7fb010101d1c635626b2586b158b9bafe410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/285e405af4597cc289cebb0b9d66a3877ddc58da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/4dfc63b2950e29b9792a6079c0662e8a46e05511.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/7959a2f8a4f8a0ebb5ba3b9d36f875ab351c3673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/79f5f06c3bfc6aedb20ae99ce4b433cf4858f012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/8f67a9a17e9a5e56b7dee798b6ee481a30615005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/e1428b2c4a7d725caa4d1bfe2e962d1ccb2f5671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/e5d3e78f71a125e05023da0d9432aa3eb52cdd8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/f4d75cdb964ed7ab40b3ee760d737cde61c68013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/1112895194c1295c7a571f7ddab724b26727500c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/1764b50e7c29aae4cfe0de9d9c5c4d770c1f7434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/1c3bdcc0e09e52cd9a68d8d531447bd7a98969a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/474d03bc2ab2a1cee9d161a112e8a9a4009d4c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/5cf5a1408464789be02b63f80d8df2e347cc254b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/7e08f7513023cf61b1dc6925cfc4199c3b8bf478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/a737caf69185957fdfc24482469bfea233e76c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/a7a73b4f3e981ad179e64d178649f3bae11122b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/e501532d81109a7008bcffd096288c42d0f39260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/1f63f57a1fa84c8e84c4f9f4e8916e6e1e69cd54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/41df3597da3fb0391f17cdbaa8d4c59076b4256a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/5ba68fdd5e4ddb957a323995392b5c6e26fd8b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/5d177aa630afbb271cfdb8cd91e6782bf0ef2c5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/666d88ef50ef0d225eb23c6d1f827ae2840cfc63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/cb881e04a36f33e3e01518b7c2cd5b5ef35a304a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/d0d4304f588db932db59031a7588632ffb6d66f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/dc898b8b6cc5b0b74a6a53d6f9feeba101055e26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/f18f520a0566936ab9e6150a91d48c817fe3324b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/039d65515e1b2d16eac15054cb0f72f9e5b19c2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/24899618f3e50c058ab42754a1c1e1b30a2aa8c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/657f2e20cbd87d4530aadd9726ce3540c3a6b70e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/6c81e039edd5971dd5c7ca173fbc0b723e7ec73a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/75b6b0f41c3c7329c87c05821a4b0e2010eec3cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/a8def138b4fde4b9b01296b67ac2074b0c7fd2dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/b92bf46326d4d12db93a3bc834831b59f7a2a53c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/c42028824ca149ada1ab5234925e9614b5d92aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/ce5dead935b4d966f7e81392a09ca835df25e763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/02b4278a38d0a197484e2f42bb858d329dc9f70f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/03d7a73af1ebab649c5e8477ce5342d8e0a459ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/0abb5ce7343a8b5b0b350787aa041c5f6d087a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/2a2543a2fa1dd3c852f01b0540d85eca7baf526f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/5fd8c5b7de627a802f2d6464218e5d0dacb1214a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/69e269f38bfffc484a38733ca00183fa7cf57060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/6d73c9bb60e9b3600e2b69e7ee31de2084fa5954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/abdb086f57319b96c5c52f96bafe49125d527520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/c1f73ea2517db91eea0f39186e80f10f5a933f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/036b3d00055542c4377678c060730506111c00db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/0418e80663d38660cd34fc543fe48d1ee1d2466f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/05ba431b00a14944dafbafe5629857b2ad222077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/077382d87cac1faffd2da8cd3dc65c7718d464c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/0b84cf1f26d9115c67f9d31925399d3c24b595df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/3b12b48dedbbcecf45de3eeef04bc1f2e4e5bae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/54c875606ab15a96ca3289014e6693dac9591f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/73adec8f8cfb90bf40a7775c443dd11ae11098cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/ecfe912d7cefa8454695088b92017d0fb5131f0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/05c81c7eaed250e255d81c5b12bc15f5c1519160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/0b401f8d19d49b52e3fa8a2846165a96c42c83c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/35ffceb70c8c0e57ffbb825c8f03c631a7ae4bcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/38df1b56ffa12c71b45ae805f22461702448c360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/3eac7b4aef086c7a7fa8932d5acdea934e6925c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/69e8657462930d7d932a4450d5fc12f9474f9a94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/9fde4897af6968a354dab6a132a44f9289e77c09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/d003660033853a5a1266cf081fb3e83b7c2737f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/e25415ca637ddb1c34c6a172d4c00779f211367b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/9ef9394f85ace694f0726cfd768db4590edcd1c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/a05e49964558d50c9c188dee721283a208a306ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/a3dba0f68f6af45575608868d8e855279aa3b684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/b5f244909b998912c7dbb7a4247603226fe2251d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/c9a6eea65a9345a4a2ac087f7b5e99fcf7d958e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/ce62fd387661be9ee1d394167ae2290fb30bde6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/db5847ef5760d46575081f64d17e098f32c83000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/fdca3516fd95a5a12aa0f83a52604d665f2a2967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/fffdb8c285f8f3b613f8faf67f211b66de11b37b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/037416d99c99f7516b56d232c13afffe0b0a7125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/11a9d73d293eeb2bf80f6481c5bc1f34c1c0cb11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/535a11f308cc372d2d28cf80e9c504f5ec8b4c9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/58fc809a0b6b7dc2fa67fcb58e9a89d79c11909f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/6cc7696ff5448ca0398269089feccfcae3ad3302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/7c5f44c8509bb4891ec12642df2cbdc97f61af1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/a2639b5adba1c44feda3cd09a33f5ec00eb6ba0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/ac418e62cd0a7369cc97d1b12bbed18f2e92bbad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/f8e861addbc38aebf38abf2a1d2a5b9f1d441426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/08083357db7fe634c9d7fe0b1ae6ed27b32ce464.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/086ac0ddf7858a62845b2586c0c3adfd2145b403.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/69a83eeb883a7d516dc5d0359e675f3e5dedc8cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/8d1bc28197ec5a11b6677ad649f12223898b7d95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/91dbcd4a18c82b6d1a65c1721a30deccf9e37a26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/99ce11879854d016b10dae2272d599a055aac4f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/d360f375f8c683ea48e404f895b20ba65d96b397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/d7bca81f9ad2e635c9d2d52f5aa314fe95981abd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/f965b206e645ef88ef562e7f8591a3669a510524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/049961b3429cfb34ff2b12657da2cac1a32db652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/5f1fbd2a21011fc114d882c5672b1261662ddaa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/63b13e41869501a51b3aa005d308151599ba16fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/7b90893d2c0869d75cef220b3d657edd6fc1561d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/87701adf3d889275a3f5c0e323ba74cb1ef5d5eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/9620d7bd3b2b91d4a1787b3561fcb22fc32b9087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/b2716c98d8ea767460d8bb24ba069994a1552885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/cfa7b81b5bb5859879160f7a9939ab54a35e9628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/e5126e746575b2cc71e169e5c8549dfd144dfea6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/18db502960cc4e3dc0c1160806853898b68005af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/26311e18d27c0b43dbebf7ca4a4764604d47a7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/af16d56ccd169ebebf95644447f1426ef0f14060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/b163aad5301d9e3538459b45c3f5c3bd31325bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/bcebabc6e7f68c7edd04398f2f7d72d4de0a66c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/bde21d8c6e10d7ad1f32ab1ec3148e5d5b5387cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/e0c8473585f9210ab0feb15d791303a5a4fe0977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/ebed4e1446a6323906961333aebfeb7f282616e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/fbc6ad166477aee1c5dfdcfd5b62d95f03ce0575.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/2add01d7798379864f161a0bb1b5f305f59a0ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/448bac4fc8d024519938347c9824a2c57fa3aa58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/742067d17f5127f91cc484ef8ac56aebd773a62e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/a16eae2640744a09235a717dfa8bedc3ecd62af9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/a595400a484ccb3c60ae7c1f9019203117b56676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/d61e431e49593bb09b30f3d3c2c57fe5d383e52c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/e2693a15c3c42b396bd4bd4eaccf9d3a0e0650c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/ea1d8c2026262cf1dfa56b6d9f0fc3b5c4175fa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/f9590f35946e40a80656eef18bff36cd512de712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/1e5d9a37741cac313a669eb331e857354b1fec42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/651a196ab3e8d424d961e4609e28fe511791f897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/6e10d9f87f96e0dab2dbe64dd14028f95fc14eed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/8addc113013a88ceb9dedd7422d8905ed2029f30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/8cf735d3b3aeeebc52100e6db3d9e0e28ab571cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/9a4d1929c90c34007dd0e7d9149fc3c135752b77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/a095d97831bc2c984e8b713ef1655b18636ac2b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/dbf03fc05d6f00336be5a315adee109b2d362a17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/fa284cb178d3e38e924c22a056d9e43168f85d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/19982ee59cb117bb426a824377bb3c6b262a66df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/94757855a4995e401ebe07dc8822dad8aef98be6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/a2919f3a0da5836adb47404da0c9c7ce41493bac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/ad6282d02b126059c3cce36f4cb1159a1a1d9f29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/b5460816ad2f67b48c0b6121246328d9407cdcfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/cd2380d20ec210fa718808ba5b3247df2c39a771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/cd9724fc4f179ad01e1d77a3ac0c311ae15b6613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/ee68414ba99ef1a5985d657146ef48c2ad866782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/f725736403a7cf6a2690f45fe06b60092680d194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/307499a47fea51020e742fc5d6c485f6554fd669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/620baf7278b0dc0e094310a3200418407966710a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/814be7e1cac62caeddf0f251206a66e2022adbed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/880d0bb03a318ff05347d3570191df3a33f3d46d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/93a0eb7cc337e4e37d3ef33cc842dadb3398d75c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/995576999077e3be32c78ee0da6f765931d314d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/9edd7a16bf0579316747fece4d5c33f350d9dfd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/ac4b070b73a1e9be42f83876e41dc333073606fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/bd8d23a81a782cbe4698e095a0f7c2323801a639.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/28685604e467e76f2f2c6d57653397f66b06844c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/47284d8c98c93455226f18793ab393e0af8556d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/5c7b3d5189b84bdc8d470540a2dd5d79a221c274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/86dbf60192723603bfaf9b7949ef88deedf36aaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/8d7a7ec87b3a3c7126310788c7872d872c9be25c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/b0a3c63c041c511f10c725ee541289974ebcf8b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/d65068fc327bd48ef85605f973b177051e3823b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/e424c3a610b8ea4647460c6b677b1726cc11d518.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/f594bd41585416703583269c0695f73fbebdb876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/108938a265b54a4746d38d0b768d4f3ebb04a8ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/279becc5aafe62b441bc9d46e30c047eff89e229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/307de43f9087b8fdc8afe17da13306c270aa5644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/56f42ae4cc38e6edbf1080336d702eb438294c36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/8b50c54a7224cbcfd80ce2b771fcd9723d076e16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/9203d8600506f28f35712f19675f6908b8f8599a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/c36ac8a4d492ca3f47e3981321a4db2908305d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/d0d972774aa33ae1685744359074b4b6f8d786f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/fd40908672de67ead916fe61bed723ede87690f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/594107a99c3f2d07bcb3bdd0d0de984e297654ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/59fde9bcedfefa5927ae1ab7f92da5e8abe1b51d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/6c57bb5191ffbc07d9c7f5acec6e01d1c46a03ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/c05fa01c6c536c907148200b7f846fe66bbb9339.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/dce102e36363a6d5affcc72926afbf9d26deb35f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/de412d8931b67d6628d24e770d6ed0bdb843f3cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/e4baff61a68aa6af5652a74288b3504231783afa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/e599c51bbed9f90570af1ea2f30c21cc80b6babd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/fe38b864d96439b29e1fc1b79c669dd9ba224b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/1cf6b9c60abbe6dc6518e2e7db7588e1856e1860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/1e9c0362e411f79548d7e86ae90e72c33e16a532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/2559053a2c211f575fe0305204de7f254f007f59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/8c1695321aa841d775967d9e7caf9c358b47adad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/9898ca46566085b422d774a6bc74fe049e8fa05f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/ba7300237158f629140d66852d5cb9c2d3d300a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/c6faf47281adc7a503fafbccef84d7d8f892a397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/c77904d689b32cf0bee8fb36207c7e65e55d9f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/e50fe8628a7e435e73e3112e123a32540569a390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/0254331c71a4691fede3e858b8101c907dfcb5c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/56bc1828cd975ec9eea4e85a0a7b9546a617f29a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/5b994677c5231c6cbe7ab36aab6587d2d71a62c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/898ea4a7349c6929cbf3e6b0393b0596e8aa75b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/b98904613871051d7308c3a124c10c737c7f8db8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/bf1acfde3fdc2bfb8cb765344aa0bdf53e7d1bc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/d6ec78e1075ec5fc5269ca9e27a044498992e2b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/e7fcfe6e669c53a4e834c50bb0bf70b8ca7e672b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/f569cbf313d3fa00123a4d63bdb9a7b10681062d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/4dbdb689c1728fbc82f226c48dbe2695f3f2d8ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/55dc5f5ce89e2b7c8e8539623ea79735a77e5787.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/62a14331e55ba68fa944b9e5fd8713aec33e36dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/6e2e9ea1192c9d2b30b2b7cc49ef04f5d20aa7fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/84f542a6fcfc4d7b5cfe779ab74e07399c384124.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/95cf7228aa823a23103b0d29364a62dc775400f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/96d2e089b0f0b2c279d3713eda4e003e5b0481bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/dd251a59c84acd849268b2254e5368546022990e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/faef3de3ec46415268cdbffffbbdfaa86a96e492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/2e3dc313edd134d0356703fe70233e6023e05fd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/351d312c8d24f9db98e5da6c40f14c880d609de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/4689f80debf91e620cb07630356e6dbbdde12df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/58af8845dfe173f117669525444302f105df09ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/70f48b70e539f5627d5f949ebca48a24930c7d5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/9ab445a30e509b879cb1467ff17aeb856cac95af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/bf632f4155d619dae954861f602679a7604980ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/d0bd02e36cdaf08d39a9a134acd2f634dc216525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/fce680141d1b91f2de3f1b4ae65b3abcc04a4d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/38f4e26c46a0b105195c974ad1453749f707f834.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/3b48558cc6d1be3fda15e9efbc9f39c065b0f8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/3c6bc7cd5ccd695d2874ed38c78f4da98c117332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/6c595e0398cec7f41754ef6dc596c6c31eaf62b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/754d46ed0391f107c0ccd55488b01e166ede2027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/9dc4dbed3da7217705c0b8102ca18127d5f2cdd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/cf59b11d883570ff09e38ada55c94efcf82c407d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/eb36111ba059ad7bc44b195579656ea7fdaa9676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/f4ce4cb219962c3f0185e4e99aab92b2d930e623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/060b046732c29cbbd71aad1366cf573000bc84bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/17bd1b91803b2eced2bb06a84aeeac38422ab000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/1952e34613df9f7f46c7135867982afc737b51ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/25cd11f2be8628990fbaff929e25002969d29998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/3d27ff8a0312dae3078dcbc3edc62e3175bf187a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/42883e64276fb7af9a04eac3644aacf31099d564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/6eb440c5a69d821aa91ec0c5aa170bc1730098d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/b31a14731e2344703f428872cb87ddaaf5fe7318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/e84746e737592a9b43bae39209e0d7c7f8dc16b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/198392ade2cb5522144f2f79df93ce33d9d88742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/1d0614d123f54f1e6564d7326de6378cee409be8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/1f6bddc3022e19ad95fefd65ef9c6ba44bf35ee5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/5580a89140257f0746149cc08d44798628f6253c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/6b52099d3f2cbf60f84c8ea1574ccd41db56a0df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/90a7affe1867d44ebf784529476efb1032c70d9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/b3b49640e4ba00f8af41da8d90d6c284d16fea76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/f4a582b0837c5bb9e26a8f0ba0fe1a2abc99f764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/f926c02552a81f0e56a9017e4d766296adaccd5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/0c734451cb26e5478fab2739fa27083040b94674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/24e6add8255c7665e9d8b36a744fef1a64509022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/2c8cd8a3eec240945c786f9e205cc042bf58c83f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/446aab4aeebb5a802169a6bc76d7509064a3a9c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/4ac23ee4d14487c9c31e9c04628aee2aeb46efd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/825e77859a3abeed808fdd010003cd6b7cd5f020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/d7e7e366699bd7a87ec137df3d09c72ffb1a6329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/e330c9b2af3297be5955cb530919cdf02cfef662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/fafe81ea1d417c40dab7598d43b23d53e0744cc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/19b55b305e97abdca5eb3341e9dfc0d2a1c58ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/1fa7b1f3f826d02917c178461ef8b4046b35be13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/2bba6388c606511356fdb4bd25ec74a5a71076a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/41efba287b9556bfc6e7c49ac38319b555c470ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/660a53920db12915d8eec90c3399b3e1230948e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/8b7071281d1caf82cd004ea7599da24cbe946455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/a3869cdf8f1daa46c852f996e440a2d4dc169c5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/acee031df070a17b84b9d776c15650c82ded436c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/beebb6f976fd21b5fe216fb2264fc0e1341e445a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/546b4052c33c9d8e2f3f6f53849d4df27922de1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/560c9b6884fe3ab548e161cda858afccb8426f4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/7cd4fcc12ab1f8a0f77a58964af63ca2e335dac8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/a670afcdcccd1468eadf32ec1d640c922cebb39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/b6c76c8281af1eb94985243a13a60288ba1c2fe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/c2fe4d2754cec5b38c060f03a2cdcdd9f9dd91e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/dad5ef4ebcb048aa64df199986461c42855af927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/e46ba7538e0ce0bbb9eb3787969dc84cf958a0b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/e8e6cfb9e96a571823b3ff15150aea85111405f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/0118d5658322cd809b8caf09128c484b19ee37aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/01a87f0516e68793d93e54706edeba380a6af6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/5201a0c0cb83ff0fdc0ecc9eded7e631c238ef37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/6640666def1af7f5011dd04dee90b5f9c6032aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/9b4ace88cacca091d7aa57594ea21f60b12fcbef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/a9a63fe80dca36e394c2674f33555f05b0221767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/b4e0cde2d7010cf35e55bc4b2a1fd7e4476856e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/c7a008517dc7d55064f79d1aef143eae41d23981.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/ec631ff5f7e3bd759e5f616f54789d3cd0a6290c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/163c83e4b011eec3d86f9e71169acbc570fcdb56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/240525a60c33994f6f1fb9bee9b53ca0206921ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/2c1bd8dac59df0e1fa78999e0eaaa1d20d03e0b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/6b755413da01ffe7dcb521349e27c2e9534160b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/8f24cde80031e70615335f83c1f32c5d8d6033d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/b21f61a5acc51307e179f6db3a558109f3f709d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/b48a80c3b36821eb2b7e27608620ad32041d2421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/c595b0289546c049e27c24f7d3cafe2e7edf2577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/f995ed8dfac3682a934c60d1754be507f85c046a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/0120908f445e46657d700e30388d9b1a582d3850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/1c64bcb5387b104fcaa0573e0cb667fbaf87cf85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/48b655583011eb9d8aae9cc1c7ac1f55a2ac50fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/776d1b082c43aef1fbc7cff1908dda6f3a693a27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/776e918e3b6d9d760fd9a98439328a9fc3c372a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/8bcd4429cd34080138365263d1148cd58d8ed234.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/c04f1acf9369f520fc7fd0ce9dc4ee62573ce6e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/d45b1eda24b3620209ed7ef750763b4b32203a9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/e9a502cf7a199440d639bca77e2b18f85dc66505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/1751454e60437c2ae8ed1876c3b6dc2057f3e33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/1cb85305893fae1ce4dae3eac862c8b6a759c9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/29772e2588afbb92a62b41d29761dc3ee3076f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/326150b08099ae716772b8b156c1ce82122f2963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/3881e1f2afe1a212be89a0350de99ae687176c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/7919b4baaaa30611b81dba30f4e7d31294979b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/a91a562c5ff172dc9aaa92752e7e9b6509f5c748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/b095a02424ab935d36706db9dc52c952c76402d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/ccb4840ef76f1b18021a4fecafdd57dea26b31bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/1926f214b770035000c2f4b048c18154a1fc6f51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/1ed7618e57c1a117c107dc801f76cf6be731bf03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/2a4de166e7c54768c39fc357ace56aad39d2aece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/4adc19b003bcba5052095d47e73f1c528734d290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/680daef1aebfac9915ec51dd37f58975b0dff458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/7d1d0a3ea570c1acb81630b098ba8e1b2d846f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/d4d8b9b68b1ff825cd0430325200faa063373c62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/e7f667a1528932555d384760496dc3a9e2470564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/f4307b69f7c86861423fe1c5038b33239cb9d4c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/1c1dba8ea443df4a981a95880d3925988bacc51c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/28411118639a79ad641ab4e6e0381495bebb9256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/2cb0a91321eb139159cb67e04fef6845bea1e397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/41da3c0b3ef2d905f9db4e66dc115645ea5ddeb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/538cae2548beed09751e33c505f1c2e915bd3fad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/7c20dff5f9862f7eb49e320677af317d3cccb7ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/8ba778293542ae6831ba292b5406c46549ab8f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/9d95ca8030e4b48bdf353c5b862d2e2694551642.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/eeca111cdc092367764b5233e4e127ccf85e6060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/13f4e6aa10526ee01d5ad056a960aed1fab042fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/1924f5e1d13f9d7806414b050d080f26515570a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/646c4e447a23450aaf8a9e9523140fbf85d15c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/7c4e9302e40fb87f6a81d2ca153c9e67c95793b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/97e4f5f9d81d03d844e60d8f19433b93dc66d02b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/9fe9a4d847616200858ced5ec7963199f678d7eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/b30bcf6a509636c21e1c35e3233318fe7ebf86c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/da55f1247b927b42669b8ef1df7a2f15e3b4d616.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/f8aeb312b4f46c28a6b9bedf091314045d4500cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/04ac9398069d4cff7fdc199b9d11f8eeb0836395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/1a061eaa9846079a3fcb5ee49143aeb1b3d684c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/1af9ab6b7eb59647f9ee930391ae045efc023e94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/4b4cc305a7ebe71fc33b2c22ba82c8cde431eb2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/602fc710082b34cfea778b8906f9e2096e16d91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/917b7ebf19ad19b60e3d0df6ab4aa681f2f91b56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/cc147a82ef49c9fad49314862fdc79148a7091b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/e5a7f8b7d90a2cab6efe24cfbbf40d5cffbf0d09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/ea98b1d92d5fb8774a11560dc52496eb9777839c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/10e5eabdf1d1912cc88c4109a9922a7c8d69856f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/1ef50419e008f2ca4d4566ced8b7f8becf8316f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/208f7e99fbeba1ca192bd904f85fb8869db59bc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/5a813d07e1d3d4582980b95487ee2ab06ba980e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/6a1596e031db5b6b952c764932df536f2b16ef2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/a4bbcec139292e5237098d8feefd311c0d33c8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/b41613f2e73cadb148473d3a42a68a3353d817e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/bab80427f77b70c3f71d3c4d62072aecd822b3bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/f92c1ff80888e4dc589d2c8152e9be7259841aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/09096521badcd7c033d75a7871eb6023bc9e9b44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/33433d66073487875f794b607a19b35bbd5dd312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/4abb5bf113744378f230ef9145fd0d4fce635a49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/505e8c92b7d00f1505debe68e296064032d0fe74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/5d51fa69af96000dcd91e3efc11b154e2bedefaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/707c7ea70c695ed5084f4759352c4f075cd1c731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/7b6f59ab554b4481740782c7cc46563101dd34da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/b43ccccf0d49d3e8635a72fd61a09ee72f6eb032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/b959945fa126af5d94cb1b9154d3f78dc4fd078b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/1af7f6fe259f43dc4f30fdc57b2d533fe4c53061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/2f828555515360727f95cda9827a41031949bb15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/2fdff647252642ff08ba4698eb2a93298f4544bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/3cbd77b49e5c18af7b0d4bdbbf35d512a3cf8f26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/4419dfa6a7762305f248f0c378387f68faafb681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/4f21347172e95847dba6eceb0760eaf318c550fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/6abfb37c4a2f34155d901dd78591ad61cba712ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/743f8111433f89e872587caf5add3317496f2fcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/a3c19448aee0b190f5daa25fa24d048338ec1799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/19338aa0b2e1cea71a964271c38a815e1a4d92a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/1b815746569a35941b0102558f356cbf7120a0f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/7bb4facda5bca00e82fb601472cb8a867973e11f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/7df035aef7dedb04382e50d2d13e8623ff206112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/b48e29ce448f62181138729df920df59d7c87b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/c6a6668bd016a94d3df90102e56263be4c422fe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/c7f5cb4338f4da11579502e3685029cd0dc61faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/dffae131e985f0a87c765af4051112666dd076a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/f465818c3b193bec53560d9315bd35aad655b0ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/1f449771d9c2cff51f6cc2963ad9ef666cea7c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/355e0d107bff5151cffe0d9045ebece216f48901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/48a2647b0f952fee7aec2072205f4dfe00b8bc83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/65d91f17a07c56272a4cd3af57834d6fb815dfdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/6e6dba9f9828b9c56f2fbf490a20703e23b832ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/70c8ed0fae5f7421aaf3468f96be3407cce0c0a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/90f4ff2c2a333ab4ba5a9755b845d712092ec14b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/9e62210c802f88549fe7e9954eb99c658fd8bc04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/fcaeeffc7b20006e08cf70eae1352eabfe916856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/024b445880a4249cf84f74f8ab1a9bf583db6b98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/23fb0ead329f6a9e28cf1acae24cf24642b97739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/40194f87c52f008fd250db2993dd3a6565ba2599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/42f556dc1749b7877037af3272e9920a9a50d39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/47cf663057ecf24a1dd905de6fc34afc19e20830.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/4f50ce29a01e0b1418218734b33e3134add92c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/526342e46e0b06d7e8bf6cb948941a3338b12750.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/5a4f2d563358fedde9929ebbdf5a68346ba94493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/5dbf28c8a3cf50216086abde723e745d505fd65b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/29c1cb67989a93bc1cc12902fa23db7c3e85646e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/40baaca935908d0bd5f44ed8905faa208d06dbe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/55cb39eff8db1eee132c563c77592aafc350caf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/5fe058d3b5e5097d7c6dd2b2edaa078773607c16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/79ed880a28d705899ce6d4bb3793062c1b441256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/bd4594ad5ef9d614ab3f0b9e151164b4b43d0ba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/d1d6cc498ebabff820334b64c46638ac7dfc3258.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/eb24df13b297d9132a56a27fb59fb739608d4942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/f6e1857dcb34884211ba91162019ae99302f3124.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/0274a0f32748c779c73aac5dbf757121cbc6dbda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/2684b078d48cb18a88dd0298e16e760845532ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/2ec029b91359764f7135e4391044aaf7950937cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/388ae5cdcbe58c521b7d386563073dd193e267b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/4d2015016d24877235b773f40eec734624c4eaff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/5a8b31a4ac3b5425d2b7b407558c04c84ac1e509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/72dc273752dd918fbfda32cc0352f6f90509e6ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/b2ed50e262362899f44b6bd05b67c68715800939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/eb2c7a49ac737c526201b749bc3c478917aa8423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/025c7344e5c18a71309aa9784f45551491fe2ecf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/25863b02f6ed757551347eec6809797fb4851079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/559578ac0b34e1ab7fd3e88b2a4443af81157dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/90503d4bae4510d5effd06b1c533e0928094e5af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/b430f5ffcbea77bace187c74bb8c993a207cdf15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/c934dbc1c51fac34246b5b8ab7974027150073da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/d407e9fd937bacd850d0b850c88674babb00f449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/ed3070f6991557e537dfb1302980b5eff2b10b4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/fee5169703b7ae6adcc193f889765254dea5b2c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/264b8ca2f5a7b1181aaa050aa6b23b80d1bab010.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/3fc8547a6e17f4cc93d0e943c364aaa15e28a084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/8c4cd7b9207a26207bf34fb807b63f5891026594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/9e3edcfa143091d7bbf8f2f7477a3e8371228143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/a61b15b7cbeeafa9ba2f128c3ac808fe63577b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/ad0256ae626e92cf15e4a82b3a4c146b205289de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/d820d2674d7dba3cee4133f64b2e5deb327b380b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/ef31669394d4acd5bbf8eefaa48b416d351117ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/f011c678ea059bac6fa7196357553da1a929601c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/3b6ea220feecd45d681fc2f1e20e1089d3e9d8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/5f616083c8850def853c4a680884e24dad69c146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/85de165dbc5532521a7335c319eddd4937440634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/9beafc739af9bce3c8e4cebec8a953afbe19c9b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/b2998f8588d001f579c2ff24eea9416e53a23dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/b2af14415db624941a924a0581233127a5091fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/da3cc1e1c150fe2e1523801064a00cd0297faae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/de37a6e23ecd7f3380132b3aff46bac765338509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/ea422bcd7a5b42885c9443fe603f51e744db86a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/8a737328d8df5f6505d03206feaa63b9a5d4006a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/97a2f4d8aec558058b67554f7af4bce30dad4a92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/a715ef487169746bcceadfdd9ab027240bdef153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/bdf22aa9f7f8a0593c587ae8acb99faa4477ebfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/c32dd1388e0d660d40050ddb80b2220f0f6c411d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/dc8379fe78637695d20db9f1781e2e92bb3942e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/e679b66c4222f4c267fb03448a88965715438d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/eb53ecdf0ca069bc6ff7c1110c2c0f8c3e466ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/fec91e8262c1d96a1e24cc41052f05c4508e23cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/075d541382008ce4844329ef526009f87dae0aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/0b33b51bd6e176ec44f32878e11d796dc68abd12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/393628c2945b5747d77bb1d936a34f08efa72ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/67877e673d62454626aeb2d750dd0358b578f9c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/bee75f9bb8e597b76a0e891d8e3eb5a2d163b55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/dbd596577f304feff9f8b532a6125c8346e2f49b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/dce4445d3ec222cf0567d11a594ca14a23163805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/ecd9286bf383e0306e433f5edf531dfb367b90f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/fa40a06fb1c3202de4451dfc1cb4b722c9067ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/22e442f0165b268c735dd2bdde7d8b3398cdf177.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/3233384099797df8845b9af8e69b7a3043248043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/3ad2305b9f7533b130fad125d7d89174e2d7f9a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/525ce85b073f88aed3dc5954e01855e16d99db06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/68c50bf5679a3142a0bfcb79a2003828d7160c1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/8de081ef8522741ef3f8871197716d7f06176f5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/bb68b896744ca49f1ab8e513eacc19b36d5d039b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/be99890a711e14a15b9af48d02ca1ec3c2d322cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/f7a363155a35c930ba6f682d236b571d916defa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/06956702631ebef1cf487ba5a3b17d9e116fa264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/3a0aaa9490ac6f0a2480d8e2d17b4efab02bfad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/52ee3dc4013c4ede297d783f8230ac672b437f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/6ad0789e089eff36c11530e561357396c550def3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/76a3a9fa8af9e9346fc2858a91d5548198544ce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/98fc857ae1ee05c98ef089e0e21e38ed13bbc34b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/9ac9b679e8c7da3a04de87e95c0ff75667c6edbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/b52270b628141d7c8dc3a26817b5c6fc940d40c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/e5fd8755395e7f2c92b154ffafdb6fe15c39d061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/103376e1a0d22fe48862039849021fcfd89688ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/26880aa981ecd293da8c6639e7e34bd7aff362a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/2fed686dc3bd647e0b04535d39c2cc9d21c4ddf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/4e3a17efd916fabc27f8483025d8b49d3a637762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/5a2215a474b401b28e2c272a590da931cef37013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/6b21f98607e9f87515d2ac11e5892c49905942b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/993535557ebe012fd3388ca163ddaf3aa3c9171b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/b55c0c35e7e041ad1bb2557c9e0ca6adb2d68d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/c840a83ea5ef0d56290bea30aa49bae2a7f51a1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/4b572bb864e28cbe00092a85e6e0233a99e0c6fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/4e42e35b18f0a5bbb3b220875d8a80e1ed39b417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/5b87859fcb9517c98b17e49f3954041a84484f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/6177a15cedd4c5c73585cf0b22439be20299c445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/807194b8af6bc859c7dd703d01d79140e8906d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/869686e83ea00c8e5e6f0beecccddc32b8b00e54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/92912b4a6b100e0c39d0ddeaf76e0bdb9987b31f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/9e7de07c4706a83ab112b2a8bb04a82d07a9c97e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/ba2f01f90a181c354ee2ee1639828d2688b5dcda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/110643f6eb5a1d6f7b52c688d066cdd5d70c5767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/1b110d7b910cd7bee0932d7f6c336ae8315eb472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/2571333d1d1a17522637f76061963b10808ac0f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/521e834ce3ebd2d84aa70fccdb2236770b3d26c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/98c0a99986ec99374469c2173d53e3138b6b7a6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/a4d2b4e5897278f7a2c061b44fcc058e88b4e187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/aba3965abf6c33c70f30749af57f7bb3dc215f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/c98e00acf270ba0ce7bfe09ee81d6dd6ebc7a344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/faa1166f600c82df60fa79a5de4d05903fdf3e2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/16791824bf5b24ebd2561f0086d6b51b1f35c9ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/3a47199b7659607f7bbebe2b8f83941e3dfcd9b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/5e9485a11f86df621e7a9171a269a1f4482777af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/8587b6b494528c2efad80bb58563ccf73603f172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/8c447d081a2d14dcb7091536aa1f04ea98f6d8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/9a113cb41bd394cdc5753134372957d2ec9ea9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/9cf07b3d3189fc2d367e78bca2a13700a0d01f66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/ce62d367c12603c9ac8f15c48e803bfcae677b9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/e516d75d122600b9eb7029737a07badd93f29e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/1463d07554bfbba2dcaaf363c525bfb5c09199d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/29e0fee0eb776cf17c15fff5bd08aeeb25d13fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/2b0cbb4cd2d946d0d7c992af67d2275557516e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/2eed0c7d44579963c57948b30fc5936e27eee916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/33774ceb040daa01ef52faa8e14fa54afd9dd733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/41acc35ab5d6ca45414c175cc86eca8a9df6f772.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/47fc5c2a07186aaff02db4bec565013efcbb7571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/70919ffb5392466b0a3fb4daa3987d36458b0cf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/bb018ac903a2d043fe3a9a5bd70ef7a8fe9f8f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/345b98f2c704a5642d866ee73b8ba6af5df8bbc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/408a020830aaac5ffb311cb20b61e94d3eb0d57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/7d9f228e53f6b9af56b9282d21945f278dae9597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/8b2443fb58f4922ac8c99432a61f26c031df0e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/8e0a6c3dfb182d2c2dc89369fd820bc20f8d5239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/8fadc42a3305545f8d54db46ccfe10d0a960442a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/9d508d47ee24e81b452d4d56d34aa5158e518549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/b1e5b67f7dd66ca4a990dcfe0aa4d57937affe8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/e4509168ef7da983bc944e9f6ad675620f2e33d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/3c93594923a10d3ec902a5ef86cce31b4338d2dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/53506bce7276e112f81f8ee7a906a11654bec10e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/5b380ec605fd5c35fe5269204f43bf4496fd3de4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/671d75e8649342b455053f6ed2e07ef5d92cbbca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/90ade5a7ed8da904e6a0c74b4ad63fea2ec04634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/bd309aea18f3d85edc6ee0ce568dd8d638ca3746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/c56066aca8756ca23d18d552d644f8056c1e6b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/ef456603aca1159de27115391303a682db3548df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/f1c802fab895d9dc73fcc2b1dbc264b307c074fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/372e9cfa4623d3ef2cce491b0b6c8185f04b58de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/3fd42de4663c575b1a3442641fb32e7a080ef197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/4e76459c741d37fc1ded5ee1172e183427087483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/61030bfd9b1af6774a7b337f108c0396de348ad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/6b0064b4e8a913a8a06b912449be759a430ccc74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/71f2f809a12f549237d2786eff25b8d6cc438b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/9ca8e30c141440ac5fe2352be4532d96d6d177db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/d9df461b3140a80fab7ef06806f234a9ff1ca7a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/eda1a5e4a8c3dca0a151b3ed1c02785513feb5f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/2004818d6fdf59681853c5480ba47f77555c0eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/297e08e968e6e95661678c4b146dca1e800632ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/2f8e873fb03714efe48b4869e28f18524d84e71f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/38837bd66df27b38e3120bbc0fb036a52a6e0651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/4ab87c754c789bb37d51fd9f57922a52ac6be07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/aa47014e144ac5752dbec75e829ad9080982d4d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/c1788152899574751f7288242931eb942a1b112b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/e6b653d3511ac79c98b5bdf442b32d52f1246a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/eb9dd004a33a114907f889b5b560745fbbefef75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/03fc1eb567ead9d846c82800b16cb445b68ff41e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/4b03692738a61fded19427f71f61a7bafedcd27c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/562a01ed2ff9857b0d425fb54e7645d46d0e31ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/578072c47c17dcd1f1815d0eaccdc8d15efdb194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/58cd63c7503aae56c95267f1d8bf7ba469d332af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/842469a9941bf175b1f51c979a3aa462a3544805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/a91c3158f14e5fd33ce6306b1e08757fbd668c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/d5ee458a756944b77fe8e063a72ff4de11286640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/eb6f393fdb5961fe3ffbf9a1e1474820c3875176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/2a9dc61d3e2af58f8354c1a2eaa50873828a8891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/3af4119aac6cab1a00362af2857dfc47f3d273a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/5ed36f2fa3546e55cb54ce5476e4af1fc938fdf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/6d0a61742c85e5061456f8120d8619b24004c543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/78548e56336d351f30192dbed772626051f86945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/92e0c59513a046b4d9dfaeb2ebcd52edb4043023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/94a893024cd1b73992be0d502cd9ce0557cc25b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/b1adc6670dfbe6c56e2494efed5fce8716e4769a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/fe83f6ab07625e7328dc6484239688b70dd6d47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/18d4ac886cd4344b138b06ffea70bf1e0ca1b5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/3b439eea17b200d684e5d68c1fdcafd1fa4eeebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/43298d5ba8542c88dfb6e29bc04fb029e18673f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/44c12aa6ade4bfc16d308268bc93ba15796f7000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/452819037bc2919f3a23ea7b8e9962f9985e8341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/85fa12f97660146d4b4ea4c4c41c57bbf553bbea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/d97bd429c529c9eb655930197bb35ac432a1009b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/e2d5d29af0c14689acfa87164942846cf4c913c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/ecaafd27f43fbbcec83c2f6c8d4cbf69ee0ca391.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/14c0353661802691499adea4153603d25de56b1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/155bea5aaf771caef2fdb798f461a2998e3df144.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/18b5f8436ad87c0b2c019f5c8c2e36d93b09c540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/1964f1b26a9a25cadb4f59c50d300ff96717812a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/4169cde3f0cebf9d8a974a2171ebe259fe07522b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/5c2bd69c2c3a83bf71fed9245b21ed666b4b75cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/6f6e1c8c1acb56738675dd2099abbdeca0d75302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/90719d5e603142c5e88b7e6e7c66aedeb64d75a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/c33df80833a222d5191d0d1f5debbba71dcd1e6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/76bc89ce05e9f4134634ffd7d0109453b609eafc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/7fe214686f9335bd44ce7b8d1eee3bf79fb724cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/8674c9f4f91ee1b01f0794610048d75c3c02f33a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/c39ecb09dab6b3ee6de08352f146693425ed1c68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/dec89d5be79083dae1a11265f61971c1d7b2c98b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/e7dbc9e08fe591f6b5162f43b354c364354877e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/f28e76da679a94f38dd0edfa37f80a71e04f7507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/f39a9ac1b9a2eb1394ab8f00ee45f064bf946dbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/f7e8087ede4f2923d239b5dd6940aa4e0ab23082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/0754faab5d8bd90a6d3e3199b3c367f1b506f705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/0a30c5d6bcf7acbb13641f4dc0d23cf34dbe43dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/632e9a019885dad36fe659d3c6eec04c48c0f4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/669f7531ba879496704077a88dbb6661a624ce8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/6ee0143a2d5d28f5ef54e82c294998aeec0b48cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/70fc1f23cf3b02c38206d651c64c635e188f12df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/c545fcc70c9a5f4d257844c47d9c160710f2f3f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/e426281532bd87c12f7308188ebebbc5d53f84f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/e7167d86af426cf1f221fdfcbfa802b9d4e86889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/09c7b1f94e56734e2375307e94a7fb5173b96563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/34d61e019483d120a5cb5aa874a949e7d8cec3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/4fd787352359d3656197d38fc6fd9b1f3f6f142b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/52a6cbd0ec51eacff859ad8a78ff3dec6e6c78bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/7e51d5967b88af8f888c8d1f20b6fc54b6aba740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/92768a8d757c8b0bb2c389725bb3a8a0505b507f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/9782f091455e619da2cf5abf4c5affe4c7271b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/b7b560d51208e9db461a0849e7ce52fe166d4768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/c67121862aae9c7862fb69920256fbafae33c619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/174c821f8ffa2eb25a234817de7c931f2558ac7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/30a726a067a7356b845c1d9f17e5295977aec33a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/328035418de7cd42ded76ec4bad2dcf4fd38d48e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/4edf82c3fe76be53ee75b6f7b2bb1080925d9f3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/564c7cc55c39b44fe34dc3ed6123e513de501acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/939f2b0ff8a8d1c1d0505bf3e13429ab0df98744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/af0b506f57497f9c03248f56dc8eb1d57acc53d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/b226d1ae0894187c93b88e0c19cebd734dfcc8b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/d2ff06430e4fb601b446ad31e69d7bdb2607698e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/19b88981b3129824144b9368441913529d58b06b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/22d23f76198ff5732c2c992502c17f7054f82a2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/29cce375a2587f72ec58801875772db343a5b043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/3e27e3b87433351d9ea1a6a2d231219feadf075a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/55c20562a2de95d8f8d144080a0085aab07e567d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/585e2953bc9c4c8c3101d1b6f1eee95eea23c0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/a3f15dd6ee3f9995f76330af28545dca5a8703a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/afd5d91f1cd195796c13cbe51d0fd6fb3166a3ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/d9a3e18a77820c24580825f9943c1457941a09d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/076b89a132e790fbca0f198ef53fa25e0c8a4c5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/088c6d0894ccf4bd72f7eff8c79e7d097b7f7405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/2f59f4ade6a7d3dd2e053dd9501ed0d184aad725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/66416853205e69f77c584fa0c7ed814f918a77a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/78d20c6dda427095122fbbd19fbab06afe881915.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/8797d108ed2dcb81f77b5f8c2c37072b128b13fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/ad4ea527c95f8268f0cf178a58daf3d49766c1dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/b693d51c38249bec30d0987adc9af764d25d9708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/e5013eaa2c648e2ceae9d30cb5ad0c054d4f6813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/07b6ee539642bad3e777d5c56c59534a7a2a7a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/317503e6a6401150e005ad402593a0210d9aa6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/3a90bddf49d14b759ade64b0b374d23647035d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/790892bf391c74ea71cc610ec113dd281f8078cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/7ae5902a9c51fc3ac9f1fc4f096f507d0efc7144.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/7d2e930c5a30425cd684a1611a148116b08e1c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/a2a00b568bc75fa5b803a1736d4666ed21700863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/e914899cf609c3a3ac8debd3729304da62477bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/f83ae011b1ea9a9dde20a3658a010c5c56661f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/1850207cedcc9443ef60ec04d7dc0da07a3c1fcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/196072eb5f07d5bb6bc06af3d5086a3a398d6bda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/23e86f37f8a4ce70ea5ef816a2d1547751d4d2b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/25c270861195cee0259b500a65879669b08f61f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/27a46d603a0a16eed6b68215cf4594f8df36cf1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/5f74c69a28f13e1e06b32d86f3c6620ecda2e797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/797b0787276e1ef01057119e1d53d64e48ffec89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/99d45cfcf8c4f2c696281aba190db2df7d0664a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/9f9b2e0f0d31d4c3f7e1d446a88a7e227474c687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/43158d24e1c26b4a53c62faaf20697172eb265eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/45477eaa111b7f3b1741cb787699b88eb4a55b37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/45f6ecfb9115e8a6bfcb5aa262691d222439de5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/6ec73ec4c7cbc74a2ccc4002bb8fec9304e980ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/85d6b3c0efd057c231b5637be7f0e0827eecf7ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/b2f2fcc74c2d59a02cd94439ecb6e24ceaafabae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/c2529b05f1aa6dea496fac31996b0c15c641b654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/dbba008e7a3eaeb31f51a3b6377f13268102ab08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/e1b7f364819cf13ed1c205386e7ec48975ac6e16.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2023-50/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2023-50/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2023-50/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2024-10.yaml b/configs/cccc/dedupe-url/CC-MAIN-2024-10.yaml new file mode 100644 index 00000000..6d0c8732 --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2024-10.yaml @@ -0,0 +1,921 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/16601d1c39e9186eed835b109b5146f326b1510d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/1e931e2b3cc313555129fa4fce84d1d3e355713e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/3cc2b53e6e96c068e924070d6522b1b976ee9ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/53abf26a3fdf4903f0bbeaccd49d06aeab1108f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/979c5f44e56f1b9007e4c1b9f5b5ba9fab86a0d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/990baa29c50f47a497ccb80940b64d67a188d39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/9cd1976e3fc1d44e3d156c0d71f3f706419693f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/b60f836f5861832fde0debf8ad6ac726da6ecc74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/f954f00c9dc0a24777fc0aa8dffdf884d9750f0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/2061e0d19b9ac3510863827d9782ba6d92ea4707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/4acd4775b1f1bc2b9f33466e898c84f772f72595.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/4d8f1890f79a6c4e9abd5fad7ffff63d2d251e3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/4e097d3fa73bf3a40becef0de2425129643cf378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/4f4cfd71e0200ca05424521ba003478fab37ab1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/8351de2c89ec26d11882d8621b3603b832a93b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/9c55a889dbef2e311a1dc0644b5610a954bbc906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/cb4cd4bf8df51f40859fc187cdcd4fa28a7509cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/ed213484a597d5c7961fb2858142a11dab98d5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/1209e3733dfeeabf2c1d11bdbc87b3ba9646fe0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/1f0cefd81a9c802b7c0fbb5bb75ed08f4a8fbca6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/411f497344bc45783fa2b0752086c7039922511a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/6970bef36a56efaa767649248a7535e19167ca45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/9338334045e594f1c79cec62de080d2d94340cc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/ab0915bb9a14e0284e6807655d48e9d0943b3924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/bf37dbdb785fc9fe07496c6ed897552847aed269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/c5f7e80282c248318d9b05c2c59fbe864e61bae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/ef2af10bca7588032dcf7d7cfbd8c35d9aab494e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/2371a43cdba3d23a067248f2dbe16867a5328131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/3acb34ac7c0fe893d0e0513e5244eb1a9631dce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/3e10ea3317867a6ecfea7e1c82bda49974aad540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/46d68a22800fdedc78670bb373fb2c7a5785dfb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/63c36087a4d20133519936734d7d19dca65ec9d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/75507ac386994f649d0b96cc976a9ee4700dbf1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/86c867f4de6333650542c249333cd553325624ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/c3707864b89f4bf364aa9eb7dbdf7fd7708bbd60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/cbbb3cb975f79e88bd531ecdf7093edd0a1c3f79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/045fb795326f2880c3b042984d65fb518be4c2b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/175a3e4f97f4b2834a7c4e7e026c5244bf19c1d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/3a4146c0f45919971358061bb74074adc6e7700f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/59fa46114a3858db227ff98b6abd95c5f0688f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/6fc99f5f285f8a213986602cd1b63cc3750e26b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/7e79695e5b2fadd71eb42e768e61229393028a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/8ad721b17b1e7437648801b3cb793a0ccb8231be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/8c85e5dc18eee76aa68a27fa8fda0db03b68bead.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/b8b031ebacd47c7f6a48d0d9d353f01ef8acf97d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/04940c86d07091c67a48dbf13058ce59cad37586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/0505460d9abc951ec391177d57bd1312ca2a7e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/15dd17ec21dbe796350abbff476c9b2d61d41121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/33be255a765ecc89c01bc27941518e67fa16e87d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/a1c7078fb3ab34a7b2d2cd71872ac2f8569fb43d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/caf0230cc2f6072a9be1d15633c13679209f06e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/cde62dd45907c2243f1a184fd8b5b9d7a0e3f855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/df49745754c8efd4d647f51264dbdcfcab2a2a08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/e2f6b2d2432205566994a706061614155bac292e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/0565f41fb7d7113883d97afb4a10b50bf246f1ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/13b6b8af88fa13b912db766ec22097cb296a240c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/2baf11f35ea80c98d4bbeed6d13cab98a27dc923.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/2f0106e62b135e386fced1faaac2e767cbbcc11b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/4507126b7c40203b88a5af1f82d85324c2f32b09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/6b1484e94ee67ee4fdaf9e2328feeee64c4bb50e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/acc880fb547907925f1f8197abd9f4d0a3f9788b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/d4741cbf7eb8701b5effd2509310a58d52dfd91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/ffa3856770ddf04a985e165f1e46d8a93833ab57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/0a3ae16e24ad20fe4bff260f5e07bb6201f43365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/0dc18756e323d8ad4623ab5292e7f92dded1a07e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/31d484eb208f854aa4d1459a98b017fe09ba12fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/3d0759efecfe13d94021efac40e39c54d004358d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/4972bbba3c246d9f712fb93f0d16b62963ee1568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/5a01d0d88391b2e5e93e8638247bd157460ae7c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/6e550b2657ab83f047197c42986ee063b7359db5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/9a9c4d863ba24fafcaf2661fcdb9d94f721aee6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/d5f4e07c47c21fb42d76b82297c7f8517cd401c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/035a0aaac701d404ac6245731d1fc44d90378f8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/1b00670cd2a774d24d2f943bc1fa6a4707cb33c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/3fe4f1ae327371acfb4c22646cd17885be455e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/4d486f7b1ea3132d2a75725a1950bb832f286318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/4defa61e43a41f73448c08df82aa348028b715f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/65f7b8d8b61400e6575122c00831380caa5130f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/7563fb563347277eb1599b25db019694d221aa90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/f4d5cf7db3390944bd77e63ee819b620006fe83e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/fcef1e5473f2aa71e4a696b45b1c9c030f984f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/1d5bdc838740629d987f0ab027a1286c26c86920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/1db7294652ad51999eb62c37321d2684fd9b8da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/25155e6cdc4991854bb66226b5a2d0016527ac96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/3c7a3d5f32eb276611a73ecd80be5079b497edb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/3cc0fbe14f2ff7ecfaee877609783a3f6563f9e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/52a84e64c42a514985813d4d5058b6cc20f612e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/cc89cec64c661de7a6c0b28f7d7de6095140d48f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/d18db82afdce70f396c05db84745f99ef0a47831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/e42d14a267f5cf1d3f75b1d5ce1170e986a9ed74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/055198c7f11d1e8a4d52a75dd3d0224dc2ab5190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/1b98c04259660970e072c665c20a2e44cf3207f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/30ebd079dbd50476e5e28c8c46961581fc113010.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/32a48891895ce70c13835e2872a4fad8f9a544ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/4366226e5485ac5c998df1ddf14add373cb757d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/4eb876bd3decc1ab5df856995c976a575e6e4e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/562ad21a61fabde799d06bcb8d160798317b30a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/5bfb3f3f2296ac734ac9c3855f038841b8486666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/cecae7d13f417c6d5d0475ac4c8838492f2be535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/1be847109766b5e9b2e3454e510074ae6d368b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/3f8736d2f6f7ce2576ae4ec958dae4f136236bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/8a51faec24b00f29cd0509feba7d026e0a4c0771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/8ff821aaca9c2aa202c729e6e851c081e6fdafd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/a25edb3f6f26e667a28c541d55b2bbbd46f42893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/c76e736b7bf928f074030fa3af9fbc19e46ecc8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/db3d90c6c94349b065242f2bd0bcab5e57a619e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/f301338b852cf4c8f36ae43fbaf98f4b58f1c687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/f8763a536bbc6084bdd4a9da76716b1af2eb2953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/0671cad13e1701cf3705341d7ec5631f2062d792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/0d0b42cee520043731bcb6c1ba6e49a7535ccab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/2013eb7ec1feb4f4faac6873005957701ed32606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/33cd5a1e1d79cc897023073e6efe0e01f2478f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/6858d1b0a8bbc9b4ae23ece32f3428e1e4fadfe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/a9cc88fbbb7a74405ab5c52c8a559aa09be3ceec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/c6366ea3f0fb8bb5044732629577a2211d48e5ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/d1ea7c918cfe4939e33974838ef34a10615d8523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/d5e113811dd35aedd5ed68198b88bbcbd7c86244.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/0b6b69197822a7e1e070ba5ab246b1500545c687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/4ebef873bb3ead0a2b73b0b0a13785bf43beba69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/61fe0119522a76e535b6d6b69b9b60e3a93633c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/6327f4976738e5886a04cc4f71ecf955f43a8817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/650057e4fea8484ab223509915852414df548a04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/7a2968541f4a6ff084bef477e997298be22c8373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/a73e9dd457a2d576eef52954546658bf7c45a6a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/d758fb5684321bd9dd9a35c30d774f77894059be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/fcc17603fa50055d803c44a2a04bd0e6a02cfe7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/18901e1b98b4164de7d50060638506685783a532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/20cc5837120d45b43ea39510064b7c67fe47f091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/262b235437cc220b7acf497080bd8c1ddf7efe2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/42e95d9f7031d573901239013f3073524330a89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/6edae1bc225c0346d9fa4030dbce1a7531dd6307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/7b5ac75514fcaec56c6e2a3892c073f7ba5ac14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/8c8891ddee360dbcd1b08f10aa898c73f09da5b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/ca960ed76e010cb97c1a25c6588a5d2022ab5090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/f4ac6ed8934c1bc27c240e71545a33f46c7d4416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/087a6aa9d69b13f34ba8753ca8709138ce8370a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/1d76df83cf771f33f4065935ea29a9089443afe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/22bfa16ba0099cf2c6c40fc4ccbf9d3e446a8a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/3f7acedf505a7e924ebc8a39188b46d6989184f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/49215cf3ed82623c28504d9206eab96dca562a56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/7baa25750f4e7d3afb43f8f56ee4cb303140254a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/8542db5bc31d180db2b4dec8d062903dedf7fb21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/8dc9541b1db655ff2dfd678641eee2e88a8e8a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/d79095cbda2e4f535bd9ce3da66b728c484edcd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/0a5baca854e2f4997342df09354a2a45e2e0b6a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/1bbebb2613e628261e03e588264a430e3949006c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/4653ad5660fb821bbe90f599fc55b855024720e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/602f1a20a837e6f53678f88d5139ba1d6771707a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/662241a818e31092a2fa1d514ca65daceedaf445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/85caf7c5937d96ef91a6dcdfe6f53eb5132467ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/a8cf7a29d919ffec26b6d46093fd2c7564855bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/c016d4a6eff408b0744e57e6575763677786188c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/eb8081de478aa7002421a3253c76a06baac2138e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/020662834eb032a15b60be8d8cf536708e30ee9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/0560a92f97c011973acf25c5d7ba4977f27ad07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/21bdb71363845d43fb0301decd2f44b67b778c27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/2f18982bb0b9ad22eb50a03ca270aebb06371d86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/ba2efc8b5e5d362147d0133b08c4c060a3c4d977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/cafc386a1203cf560255e4f82a7d22e10630627f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/cdc5809e64cb83c64e008a23d6abfe9990ca473c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/ec2473a346c4d4be29b34e418f94a2280b8f3764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/f1f564355cc495cd979ea4dfa08bdcb1e581b098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/082815727269a369c8c5e0a5e9f81add2b195bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/432872ee1838a88163538c577a9574ea72d26c90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/4793cad5a7378a128a58b6e0a9555ac6c8b85f94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/71d1d89829b5e4d83b43adfccb21bb342d53a2b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/992b3570edcdb179ad6ced492086fc0085c36e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/c61e9c6df1afafc1d3800b49de8a2b86d5c19969.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/f6812b439dcf46d7881ba346b7151844f5ee387f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/f969e63ffb5a1e63b136f5dbe18c1c256d2af0c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/fc06b4ad6a0df5dd3e2fde8450d3eaf8d543551f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/00e065d249271f0b4d9c6b19bdb012d23736cba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/10509e0157e5f85ad8415c5ded197d67de6562de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/21a71dc8094e8039d0094e48dc6f9a3ab2d92fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/4edbabc57447aa546227816797adf615d0ece9ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/513e71ce1ae0295639f700aba5e74ee573868f53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/850ce06e25d242333c124aea5830cd8fefa0b8d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/9f0167e8a30730464a3bc5f8c15721658b228148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/d73b8daa4cfebee88f69b7d9448264bf3d9f3bcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/e73b5477c468daff241b4406fd2f723247c7b6ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/08638f2f8c64ad16294a4fafbe6ef8d8ae50afca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/1ffc3236c19a03b72146dfe705d72ba7107167e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/4c15db96ec2e86f68de6db4b8e94ed2c2c658b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/c370821f7a7a7e7776e9fb27a68c26a926c8dfd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/d251f6b25259bdce62994de7640a2716890e2a34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/decd359c4abbd4352435e61712fde1ae1d41a91c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/e18aff9408e28dfcd005866d7751d34e61c57bd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/eee7a578bd8623d4d338cb5f48adf96f4e9e42af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/f0dd65b90442b394b83d3a9704932c17235e71dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/63b503ebfbd0de4d2e91c10f4cb559409848515a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/8f44174f1e66cf654f77d6c60cd007178c37d3f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/994bfe4b650307f3b4bb4b9243a1e6b5b3832506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/9b8fbcf83509b1269725def3e8de7b24ced40f46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/c40014c1e89496f774f131a3b810c93638fde3e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/e6e487d44e374e2bb3f0e2c891349bab8eba1a23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/fa3fdc05043bb9b458c226392b75bdc3b6a943b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/fa4c5d08ca319d199588b87d33c0be3d6198eeac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/faadde48eafbdf8b59dc003c34f5514b445892ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/160b603873b663fde274233ccf43d3268ab8c8d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/632b54a49388573b8b7a3cca53bf0d9b2cfae6d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/7613afb175a8ca9b9c28bdc2e37b99defda10cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/80796cfbd6249f7e38159a308619ba46f8bfcd1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/83119c6e9050c188531de4aaa4d798573f6d269b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/aff0c970a4544663e61c77e8a2dab3f3cb572f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/c3001383808010e50a5f612eae5bacfa6e5d5ee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/c617caad2fb66f860809b0f289f90bde790027c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/fffc412cc04bc8d614c216dbc8765c8f480fd6f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/4da7363ab40f83dc9fbab527e64aa203e46d7d9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/62168f24cf6e5a52b699a9fcad573f17246f7022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/7a2c1af8a81c9dc2489fb9ee8ce5c901e4e4ce91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/7b2efdd3a34abf8ba0c885728c36dba4c3363a80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/97db6f23a9407684fbf4b58c1e00eeb8b5c5ade7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/9b3c9bd7d34ad22afc61e88808bb6f13f57001e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/a8ffef2ceba52e6a9232af96de1b80212c6f211a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/dc40106c83dc2f315646ffeb5160788c792c37cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/ffa6be003acd4e56a99011aeb2d9c47b412118dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/02bec34e7713982ec87a972ff20b2268aae05d05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/220936258f626bc58dc988b02b1992b33d66bb42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/38d5590034c21bd0588b7825abbad970a2806b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/74a6101623634a77a6e94b99cc3ed6c67a9f627a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/7f8ce4b1e8b573eb97ac0d1f73aad807d0f07647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/ae444aa88f4702f86f88661116e8e91508e55e79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/ccbaebe5f40053c582445e2e25dca428fcaeac52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/daf8b5fcfec217e991046a14efe83d9567a5e21a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/dd5792a7a5216f2eb94fc7a74515c84c21120489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/0842459baeccdbfa1880574bd9f74d485b4f4ec5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/29e8039938fef4bd4ba2c4a96f630dace443bfa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/2cda45b3820c12853adb7e797e376de4f21c1fda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/534779455bbc63a0a0bfd46e8abd2b6a557f168b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/662fcd808942605c62da37d1711470865d9e3b87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/80e03b067bfa92e740853b8a432e5a9f4d1f5280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/8d704b7d31a3750032f28ee844dda7b4c9d69f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/a38b3cf0d28b9abe95a2055ec4ef43bee3bb0ac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/a787a0c9ef44b2bb80fcf16a19c874e8ed951e10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/082cac7ddadad87714fa83e81cd1537731f6732a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/21b51636f3e1e603fa00f9eac4960af714c1c01e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/778f6f010f6e9f3ad9c0015004fe416e6f5ada95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/807da568e7c447794e307d26116aa1bf6c7c4f2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/bbaacb5afdef6e1378206641f0af0a9583af7269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/dc2c01ec2b7004b6d5d50f82bfb51f2013fd28f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/e77037f035b009547d2f467ab5dd85994800d778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/f295c413f2cea25d4406efb7b4d3e4c939deeaa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/fe65784b1b5f2757ee885e3327fc4079cfcb18f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/3ce592fffb39ff6abc6f3afefad1446a8d6cae5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/5a4b03ce5057c902918d8a5d877b36c19ae8f35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/6480fa8676a948b6e195c32106dfb5250ef6d4bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/91bf72bd169a2f015dbee4f77fafe3d08ddaeb15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/ab649381e3cb5f3db5ef7dbca2480b342c2b5c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/bdd5a2787f9fca9981b65f7f1bf3dddf27af0717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/c32e0a92ab3e951a858c1aec2572bb885d3de3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/efb87d988ebe3cfceb5715e57234621e00ab4b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/f1c82261a4807c376b5efc26a9defb74060714de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/4c636965b6145f5316168c177db248df2e2e9805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/5bb99a6e9833088210ffde60114a4e85795d8934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/627cb5b67475f96f209db2c5da36932b26aec326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/b6b9409331310274aa36d9fca993d3c98218e892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/c0fad2c9837df8d5b6814e9c9132ae78644dfe22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/c7d43241f78747fa24832a01de017101ec4d100e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/dba257663cc628317573c41c45ddbe2cb68c4f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/e866594487549e16172398fbd751f322b525f710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/f0d1e06fbef236eefc406deaf6fbdcad89e12373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/1666dbaa1277aac9272a4fd8ffb1a1b55dd2d233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/394de2c5809f9c2101ff4201f3e7cf51455dba0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/583cc041fc9d42f428aea8c67dee534a02db7ea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/5b42ef91efd3c32403cff18ecb8ff85e79149103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/7b27125b83941cc379380f4572ab328c441873a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/7d19291def6db0e4f315350d65c2fc8de907a5fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/9d258cc1c05e1d4119a74a0e7840445253fed3b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/bb3269bcb697e0f0d77621bda877d4a4f50bfb29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/d9b6c4f07b108485f29232b20c703fbd27930815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/30d1b95c576baa99dbbf47f297c7c9529fbf6c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/3fd99953b55dd7d9ddaae47e1dea64becffa8d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/574530a0a99ad8a108006df0d31c9976dfd3ab74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/5d131884dbc9e43ebf41542658cdf4cb0fd3804d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/7596207f65b4c246230d8646010dafdb351e65ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/7f1cae3a95720a3c062dae337319e53d060a5ad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/952f75f9136efb22131fbbdd4ade0d933302ebb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/dbd7e3d1136a6dce529b7c819976d54c5679f3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/efa95b9f847408a293d390f2aac0795fa77ae03e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/0e7dad40fc11c8551a07bbcbfcef32453559440e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/1d056ff4285e3ebe1cc0568e0a8de0ba81474292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/32c1988afb76a3fd562f421e37525e124b869e55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/533072fbea305e0a9db1bfc72fc675c33527aed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/7ea6f44c0338cd1c878667e1f3cd28bf711a6cd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/8cc4be701906a739cb4ced29f0f15ff2a2ed5213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/8ef86e77bd71c0e48769e53ee5aad9af3a0d3079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/b132a04cebddcf10a3eb9eb940c59583d3957691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/fa2d16a3e9a683c4c997f77b7c5a9bdb43c24f6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/0d4a3afe44ea518be13e9b785494e765fc409b7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/34fff3bcfbbd21bc6101d03202466a550603229c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/36f276d26ee146a33b252ae28e35362db367e712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/697e0c6d98b41012af8ae28335eb02e57e599e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/6ded35ee3c78dfbd4071e7af57cbbf1eaaae119a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/78b210902eba3b468834aff0a043b32ebf3c4914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/7bfbd34e2b283b581102dc3a93d56d17f8fdbc0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/89c02423ea893f2ebb6f6610c1a1864c7c7150cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/9e6ef59861a9c239b344d3b18b3cbb7ded6e326c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/264b6eb3350702e03400c9bb01f35370d7aefebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/5412e77d276636746f3bd919e45d0b34c30720d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/5544889072091bb6fef7f6d069a140e801c0f095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/5f2baaad5280c571c6b36f5918b77c75d6ede289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/91c637d3580b53843dfe95f0deef98067ab38dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/98fea82cbeac46484f0b8f0422c211bc514ac2b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/d090063da3ff78b5ba0ad509a6e91f866139074b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/d6330903a3d2385fba04a9532bf460557156c26c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/e74864305d593e36403a2176203e466aeea399af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/01112c633bcdfc37f74b0f55ba27c7dfcd718b88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/1e4b5945cf2cc4660ead6bd5810047a422cadef2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/3117f60f1dcbb3f1646ca8e3eec1cc02780bb28b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/3a28330bf70a4243985702e4812b63b54d30ca78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/5809482abfc50ba2c052b0b400076415fcf9d93c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/6a08d894448f334ffe7fa91370ca789678530c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/d595a40752e6036a3fb5fa3d029a99582fb5260f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/f77ede3bfb8559d90d6941bc586ab883c033723c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/f852f8b482945d1a43cc9ea2cfe656561ba145f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/07b1af0056eb3e1b64a2c2d39be0b97650898d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/1832df5cdc88bbd3d4574ae57039519076e2cdb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/454e4b7450765d00d00271a9990f1385b6afd3df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/46a502cd9f2c27f39b7dd3c8531ee5c5048c0859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/adb3877961825615cd804f873daa8879a4a83bde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/b77aef1a2bb33c1a6162800b0c362f0b5bd4c852.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/cfe2b73c04765949b8606578416e7100462c3f59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/da2f475c5a5da369cb56b9f1ee21037186c9e4b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/ec3a85cabea998a37ede6b8667bd4e88cbbeb796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/57b395b9d47a5a53991c4713e7b7f86405f3d8f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/686d77cd44a6dd868209e7d4a0cf2ad0e4d11ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/9097fdac0f2f94e3ea03bf5b53a708b81e67bfd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/9e0b8ba8f44a5cdfc8b559d23ac635710da7697c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/a099c34478b52bec9701879d52f39dcf25a633b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/afd5f008f6aaa6396e9dbbe18da6a82219b9ee7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/cde70f161099dd31a4f942a4e8cae262f3702c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/f523d77512e46cd4e3b4bb64a8ca380b1e8440dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/f665cb5180bae96910d89f0925ce13341258ab22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/5c2f30ffd411ec5fd2531b478a22e346d208981f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/86379868deb1c0384a6534ea86f4228390920862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/86675e3cccf7a31068be8363b9e385b64b25de87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/9c58f2d9fd1735f3c15f6b53cebf29651df19b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/9d150f1937f41562f2ba75802dc5741959e7b123.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/b39bd112e8ca44fbfdcc43cfc75a2f59b7bd9a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/d7cf50be695bf2cd1e217f02e89bdfa0b7571661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/e3f926815bc7cfcf5220b9f6e08fcfe8ff6427f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/e535dbb721be538aaac19d5d3c3b7dab28d4ad5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/00ef87444400b76c4c1026b6253458228631af30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/112aa1749c73abccd29b3557f4edd488a1be8d6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/3dda566da3ce49de8e8921a27ded21cb5ff0732b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/5a48180659665ccb84bd3f48687bd0a2681207db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/8e718e65b16bf0ecf96666aab22faae304e9d1c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/ad6da347b55124cde8c1588d17da8eec5cc01e98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/b4331aa3cb51d993f43cc7b96bb48c7f636235de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/c8159bb80f0d6ae2eb2f38e856d3924e466284f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/eefd536c5765c81e81cfe967a9fc7fe138809e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/1c8e4470a55719a0204610da29d606f765145c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/3a2887f46005e5d61163b23a3e8f782232d721d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/48544f8c9fe1de698741fd7d70718423e60e3ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/66882e4eb2b65abca16c8dc385319deb21b1a965.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/6a042afdadd218e160bbf0969d27114769232f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/8c9c3609ec5427f41741b1cbdbf602a44e48d111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/b8da533a7151638f061bea1923bed1290f6a7cfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/e28444192428cd6133d99fa6ef7e860745d9e90b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/fb1cdca3f1f799fc7642f8699beb21c81d0f48af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/0b0fcf327bd0e38ced3f2136fddfa6d9c4b27b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/1865db6928769ed5a4d7b976dc5cfa6afb2a971d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/2b248ff637d04d1af53f071d4525fee44b111686.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/2d4ed6d14e227fbd8e218f9e9491c018d08e4aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/b617de77fd8cfbd2077867d0db338513b864d925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/c187113e1fa87ba09df9fd800dc36e99a7446999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/e5e9bbd54a5dbb2600e2d178fd1800804e6d8249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/fab88769522de3619e8bd981015339194c82e357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/fbf2290ca893a1be9c9ea842210d2d0359b23179.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/1a7f22f095b9a190ee27d135f7a91969e9aeeb32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/56594029eb1ffffa53ef3eb52c94dff0d4f489f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/5afe653c526e35eacaf462b2335586538600e60e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/8354bbf557419d54c98453d7c8494f233002b12a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/8a0e6e8ba3af7eeb6a48ae0ce34f067acb60ac5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/b65e2297523b14e8a851f131da66e430c9ebbdaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/d39843f936271e89abee8f76d12e89e58902c682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/da754b10bd506e6a46735a32d4c64d84188fe712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/f500995c37e4d5ada50792fcf778621ca8a998cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/08aacdaa7a59060a3788a351d88232c226810006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/09e4a0d4a8effe1a29c3466fef399915804c56f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/1df7d96bbee6fbfa0e584b0eac2abd5a5c42ffe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/2195653a44a5d6d86ba308044bd40aed254744df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/6b59748ba52943e69c6e3bfeca9117b4fe5aaa99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/7dbd1ce74c1fff0f1576e3b7626aab5bce470f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/abcda5da64537b48024c879620ce2d475fe2207e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/d0fea3c92728399ae0477df533b2ba513b902d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/f3aad0753768ece3207ba6bae38c51e22fb12ea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/2cd91b6da939768a2a847fa0a9b390ea40d39a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/3b2b0a4c0575da13b2c3950bd901f543270b5d8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/534eafa0bb084b89af5755a8da8b124e5727fcb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/572e9958e49e287f1ca9119405cff266cea482f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/5caa3557c2ad0167aeb7c6251acda5584e908285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/77e16c007667b8df09e8bf1d5a429d9a0fc7956e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/78f734bf190fa73f6b1918d1c7657b9bc64eb467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/98355263bddcd6a43126283cb1c4383ac4a6f45a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/b15a56a65a142e2efbf2f9266e69ece1e1717e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/0367e33cf348b56ee32d4cfc0632a65a6dfbd51a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/9726bd0f5bf19982dd3d45dd9dab5f4cf41cfb31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/9cd4ccc38b440d5277c1dcf0ee436f90bead3510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/aca3463b5f9737161ff2035521f3408d370ce5cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/d5357f14fb76f5fa6dd0de1b27beaa90a4f75c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/daee4de61795b1a579bbadc87b983f0042534152.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/de0be6b42e4ae38afa9d1adba1c6b67fe1a544c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/ebd5f9b9546dec0ab8ea219e022e86ce0c5efac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/fc33d52cfe9daefeaa27d0d91cf58dc9e1e75b30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/01214549e26e1cee21fba8a071daebd51a12389d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/12dab9831f38262b30bf982adcd8fb092ca6f2d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/675e65f1e72b257cb604279f05c276795ed5c61b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/8d36f8e1b82fe5b7a3a5e2d2a8583246154e34cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/a8977a2ae5d85fe1766438864f507fc726bef107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/ad51b1b4aaca56f42dd99245190618810f5deab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/bf53350a32a6b73dc3eeddd3f0df9786cea6e70e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/c757a1f67910007f7f99e694877889703c9b9552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/dc2ec098214fd8d9de6cb6e3bfbdbb9841ef7db8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/1f03538f85a78234e9f61181c8ec7a09c325f903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/2d0554dc960bfe4898885884ce47a92cea1aa7bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/3deff554418d9824eb27155dff659a9cf14606bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/61d7026606b0239c19fe0798cf8c9f2c672029d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/7859625ec054c671e497e7eb58b1ce7d51f05306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/8c6c496a9bc4644880dc09b495c48ce01a29842b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/ca5d5821321a25bc1a8de045ad81715768e05f19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/d71e52bafd3e595b1f51b99377f01450784fd601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/f25076f9d283f37b02ee7b63f40f83d4ed2ec265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/09f67c24ac730a71a34eb16c6642b3b22a250fcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/1e626539cf3c14925f3ce16fa68cbda92fd20df3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/3e9b2c6d15dde2be4a1bd3a930483368cb989a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/4487c28f491106288c79c198aacbd18bdc87bc71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/5be5b76165fbc50ec9c7a369be068f645f1cd0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/74e5e318c521b5118448be27f77bb493364c071e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/aeb4de3a9e3044afecef5e7052832e184b7d9e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/d1f270a907e621db50bcaea74e7587baa41c722d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/d3a48a8725d53422d4000c8ce7ab0f17ca36103f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/037310f44bcc8bb112ad1b82dd2c0718d7ceaf17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/575c16acfc14082e98bbd97374bc0d5b173d0bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/a135a3e21018f4d6cc48f8f7528841c045b8fc8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/a29af557fe7759652682cde8b309ce702f28c85b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/c250a29dbcf714452641d01c6afa710f3f95a23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/cee05f46275a90b2caf0f1f5b27ca7b833cb9cbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/da83f73fa09c078b459052a2c6406ff2c823b323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/e5a8e04fe5dc365b7db3703523c15ffb3745ed5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/f78e5dd7c8ba00228469c357406ad3d1e7b9d6b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/2186cc81a8b3fd0ac659acfb273809cc7056e0e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/26ca3d51455ad89b6d6452f140f405481e1a2931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/6aecd7b4a38d8a4377f065382bdf8deaab815332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/6df35d53b8f7d8da8a394e05718716c526728d9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/98f30e33f1f9bcc58bb0d39b1fca2ec1d49922fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/a670639539998c56d48fdbc13b8422274a48cb12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/bbc66d9a46a420bdc79db1ad3fe647838d78df09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/d4ea7188f89a8a4d0a6da38450e0d4a663127cca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/fe9c93fa666717eb4f7baefa3308b914dcb4a8c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/0eccf35b02bb9e6827b03b89dcabb2e2892e6666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/638653101d54cd910a9878a6014be7d568b19949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/89397bb3528c3bca8eb8bdd750d8373df60b9dff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/96c4772bf36133a57ac98d52446dd977d8d29e24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/a4ca6e84f423fdf80271ee8ed6441c4ca1ebe402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/caddfab179230fbe8c050ef0cec79796d9915336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/d30fbdda431a74c1b0cfa000a119a979d86123b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/d3dfc09a7f17fb302557864f57b1e28b747def6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/f746631ca6f784705c70c4ba203e7cb42aadab6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/4fe68a4fba949a33325b27a69ad013a454451aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/60b8d6bacd17b0f893541c225a1a393fb4aa49e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/619622b9cd2d54c83640b73127d49cfd4905cc58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/743c4a9ec73003aeeb8226fd64f8144685df3603.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/a034126945bf31e21d2d3b581b2fbd4c7509cfba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/a24bda735c4f6f69b6d905382ce628f553877ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/b2162b147be259b7388e51a8fd4073f39180a23d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/db1104aa9bc93b8fe6f85f9ffa0ac958dd141ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/fa4b7b80dec10a6c7bc8556f4acc5c518bfb1ab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/2937025ee6cc9b88db7717738d391fb05fd74412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/326dcdbab58f2b826c347eaa1ba990d7387d5d92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/570796bbb46ab3797e3c99cbedd0ac3a654e5a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/c2dd1f7d71fa4f5bf36c5c69c53343c4ec68aa35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/cc597371b3955ae5fc467d98448e0f7be262e955.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/d01939f5320abc03d6fef8aa6cc9766ab7c92427.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/d1ab1fc191b3237f0663cfd775f91b9d9190660a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/ec05d57cd28688a8d85ec070968ed1fb5e73c435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/ec24d271aa1d153a9544763c544c4c728fe88004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/57c4bfc203e7d472d2450a7adc426cd6d73244cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/660e06d12e3ff674aa81ce53fcedca33505cc324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/7aa68e71a6851a8ea17b70668fa4e8a3e755e9ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/9b0951d1295ac2fbe7154583f5258d28b1d85a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/c0951e69c5b3248cc3375706542de47fce6b86e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/d9e9a8d1436914b24a0c32fa8e2b37529778ce38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/e05c95702a0cd772189c166bc50e1c1398dd75f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/f02926f53f2114d2caf73284f0382223de054e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/f817775d51ce16b51055e2239d8c4bfe4af095bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/1e17cab00aaf974011959c676536433b132511dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/2c7873b784a31fe5ea053570b682e614a6fc8a43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/610f1c2b8218719376fe2e5201bc29b7d64160db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/805c2700235f753e9dcd8da1774de00c67fdc808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/95992114d9e90e9c6b0cd5894eac77929206c867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/a24f486f3588acdbbe06dd70e497c2699512480c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/b8f7b458436dbaab3c9563e95bb36fc8cb9e9954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/beb2a941b0e46d20d372f59c4ba56670583d4781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/ee81525b1f97685a7b36c4cb8aac9e0a240cbbd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/268bae2e34294ad01c2faea9f6256e60c745d164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/271f34e472ef7989d8423ef98b84db1b0aabcb4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/9d9bba6008c6d0ff431f2f91d4b78a01acb31c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/c49a5820718a790bf9c4d54cb1b5fe85b9ef5361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/cd8d6b589fcfbbd0d26ca11f6d9224a670c0e138.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/d1f47659f700b14742f405f1bce2a431ceedadc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/d71b5aa76f3053c71e08981b5de67040f506eb3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/e2f9abb1e0e64bb93512c5195351952ce8fdd5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/e3229defa8f479c12600445c2b87736c101c523f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/3839c51754f7e869ccadc6ac31bb98a3fd154c8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/40e2188fe6af25eed5081a9c9de3052a76c96961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/4a0abdf35373128253d0a4c8e55f988efa0696cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/4ff12e65a98d646b8bb1a2c0afe7f2f6d90d2bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/b483e7a8ff82f9cdf3763b9360787efd417ff920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/b6ab61ef442a94767b7634743d7ea188bb60dd36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/c7c29c124af5f88831d5910a937f100556a5d400.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/df49f5c568a559f2325c31aa3c214c74748d90f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/f915bb55649ce52e50119c9182a929789d5cb7b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/158a0838d1309ad01fb30d74832bd80ccc413b53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/16e62aba2735ecde45c88353ac72f42ed7bb8185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/1bce03dbdbfb5f56719ec5dbc4dd57e703aec6f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/3ba443b2d4f0bc18e18b8039128168f28631af9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/52dff5252975b3b19fb47fd74e4418d2dd9e6966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/7a6d346e9d59ce90d3e2838ac9eb6ed3ce11e855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/836f9e7b73b0f774d5b3ace98e31cc72a2f99faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/94b630dc96c409ea9abf1f4c83e94060ed2ece0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/cb8f281b3f66a30d4fdb3558e347ee35b106fae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/2660ff862a89007b187cde5c39f60d700a9a8cc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/31ca94989eb4242fbdc8204e6469171ca7eb6c88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/626c52e5a640ac438ff64006977594ba83a911bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/79df8ffbfb8bdf449e991bc315468f78b8aabb7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/7a0366b555a137433196e97e9d06c69c120a7160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/7bc36bd23825fce6802361105d5c771b752a6da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/947ce5742106c75150d58f7d3fd4b1e1e233d59f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/95a48d2d802824061017a5c13e8ed3d8b6068456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/b54b708ffc61c07421f0500aa7c1f4158ea14de7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/19d124e21e132f1e139103409f8e924e5ba5980c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/39221fcb4be7f64ecd6d133fe140563f7f9880c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/4dabf8e141efc18e705d0c4e7836c630f706f28d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/574f21fffaed9aeb7c0aa3a5dbee90cd1c05200d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/8ae3a7cadab7d797b8da159521c5073c270f4eb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/a829028f77ac91244551dcca62288cd3eecfe216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/af91e36eead44068b0dd26c57b7a783cb3071953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/cb75931e01a4c0075e4687b5f46a28f6f3fc1c9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/ec3e1f8c8ac6fc9970d6ba8d726c7c6cd9157d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/0642e703de10c5a1cf1d9ab1ec6b7f1c5a443c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/13435847eefc9336b8704c6c425a9ef9e5f5cda1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/4c8be88a2bd377d2369f647984d33067024b21f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/7e98b56c2859e4bab6be3ab88a6550253d31c592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/8d67320f167aca08fdeb97a0fd91683f31dd3681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/9b2ebc5f8ce4a17b457f577d68e94bdaafb51637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/a8a240994ad4240611a32c3ca68cc3125cf96bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/c371b2ba57f9ebb26f9696cd7be7c76d00b8057c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/e2f893be615b8bfdbbd4f171b79443a3a2c48693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/1f8fb03e0dcbb5641709cb5237059c571c319305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/4c8c921acc7819c828188a6deb0a935009c8bcdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/4cd57aff8dfdef94de7a482f049acb633248ff1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/5e7314117de54b9e177f50533fe300d88452625f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/6a3587b1ddb7d35c09d20395e0f4142ca9dd5f45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/a67b34985d0b58c4cbefc25440a6c926b78070c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/ba1f17cd6924340a8b202e82b84f7f90ce0e2745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/ee3e0e5481c535c1b7ed0f7126c3ca2905fa8eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/f6b791291e843795e2cf5402b57dd58487b09ef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/053d612f99ae6d03f60856d4b6f945ae011471d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/145c8335a1e5e4dcdc0717cdbc8a50fc748b2485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/53b37285224d7935d1b85b097b19f4365d81ed3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/54f9706159e2ac989b608238669232225255e72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/5f3cf380a55a8a12523c2aea6de53a867ca5aff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/68c84f01273315d3b2d677160537568855e754ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/76d683019f4e0028b64bd3bce22be7251c7392b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/7a6aaddbe02c48cde378c8374300adad204dec2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/c70ee9c069e1ff9cc0a77997d785e61c782dc6ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/129a9a5303f4bf083efbd1ed51a5bc964b912d6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/851b67c6b9af696fe09d88c55fae46e612e5fc58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/8709ef030a3d29f0cd73196826e61c63cb5792ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/8ddede3e7e7c43d59809b74acb6625497b8b49c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/9f91f906423375b3b80a549e00f86c0512d5ef5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/ae1ccfa55612734247dbf9623ea8439f5ff4ef94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/f80139858d1baa17c359d2fb14f0edc5509fec64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/f89febbbe9fb8ed08fa1a694d82eee9930e7c978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/f8bb8dc7f5b3b9ff8746546cdd247cab32a00682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/35487b7af66a154370d2a05ffc7a5b64a5e22312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/42559d1cec69c9b4a1de458496a43e6dc676ad0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/5421d43562df4d5829591f1f3a9083a45f647093.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/5d69a6bbac26fd9f9f2b2125f0c3030e2384c3c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/9bb1f5e3335e40d3cb2f6fb78b0427b1b5a62b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/a9a7a57f2dd41a2410f021966a4dd6123e11a0e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/cb09af2740aae176656e954eae8851b97d2ed5ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/d7b547e89f7e30c880488efc0db7af4d1c7d2005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/f7553233b155d6e17693cb7c9697eac7d4bea032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/00f50181fd3a82fc608dcef55141d35b2a45bb8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/20f38fadaa9b861010b6e2b2d767fec3e1f204fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/3467e7b2d545cf3bdfd14e87fb33f0950ed764f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/35bbf77ce81c24a61e2dbd873a8f9a3d24c235c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/6ff765f984810d19fda68894f5fbedbdd71b0ee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/794d36be174490e0fd176ba6a8f20af12f4b1203.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/ab785d1e39fc6f4c25d72d98c751aa5ec2f10a66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/aeed6ad8356d78aca97c043ee3c1d4c08b36c0e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/bb1bd070f9548ca60af982407c3af0a1e83afa74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/11ce6b1bc7d32986a4ad8ff94921fcab76efbf1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/137016fc8e86a13b0fddafa0f0aaa32d2c797009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/18142ad66fb72e4e1698c041532fdbe141fe528f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/2bb43a2054cf9fd5e3e98acadd5224d4df9ff03f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/422817dd5cc7ffb0a8f0c827a516debbf932e2b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/4ea3d08dedce8e215a15742ebf7959e3123818be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/56f2685b71d1e552510461f74a9d25669230cb2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/675e7113995b247c8cd1920acf502ed17fcf5020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/8c71bef8e2a8bf04c5a43d076cd7adeed5583b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/0213fed97499aa2963ad5675589faa174af9b4f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/0cd244b4112e6046547d08a2fdfe1a6e7df2de20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/1a57d363d36738d7f1dcc2beaefa00a0d815153c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/684e2fcbec48ec7aa4a688a39fa0c0331f377e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/7a89545c5235706f606d3def2b5f6bc2004306f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/a18446f57df908bef5343577115925eaacbb5bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/a9caa0c356978b39bfb55676dc817f999d2783ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/acb5ebeb29700a478a15e96fa03cea64bc6f0c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/dd450b4ebc33ef75c1f46c193d287be7e65728be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/0959e5bac845542284d9e33485c2b968080eb417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/0ab67e61d340c849fd8de6a1cc626705578db9a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/23e5854286683fcdce359696416f5a9232808d9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/316ce38b7bf2e7ba686405dfaae2e834c860c84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/468bc1f295b428a69e59c72ba6e47b3265dedd12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/5b282b06b1961f582952b075dd006924c8e8a298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/9c38ba4a1dabf49ac136ad000dade94e9540b786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/b5e4c2d4b47c8dc0d873a90cd9166d5da0abc471.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/efdf2a8030c707dae101f3940617589c7c2f4a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/078d0540d8abf7edcfa3869ee77bce6e73c35ed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/34a659b258efd22bb117336941b5e6675d3cd46a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/50acc8518f413753e9941fd80a971b230cfa2442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/54b8218617a73f51665a30c70936d1ffa793db51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/7bc775d90debcc36d023dc95839f5dc90f4aaaab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/ab408a5aaeaf9120567898a5edb9375be3f3e070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/ae82bc7a3853d287125a3ce32fbc3474227bbb0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/caf3fbabbaa89d462820fd3cedca7caa8125b703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/cf82d376846d07a3859616791ec511c147633891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/4bbb59cf63e025f6bb18a13712604b7c3990a83d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/60abc1370a3ee1c0c97f008ad5d4cf7d6af25845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/66a5406206848f769162af8f73703d0a66356183.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/aacf74505710fc10dfd19479945b6694b4f029ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/bbb212ff1476c1bf707ce04248269c0c72f9e15b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/beab468bffcf96f3c98b379498ac8ecc3da6596c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/ce303331c0d855083563fac7ae55292509ce1baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/dc03be94a1024ee3ae0d87aa8ebc96525838edb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/f018e98a628e0d6b89837a7f6ec42ff6e19b8fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/01ced0ff6db69753438100340f3b7add4ddb3278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/1811d67d11ba9fb0e7ffe282dadcf10491eb9c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/2c7cba95298c9531e3bfabd0d72e5c17c71e538b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/59f739853785a525bf3d87f53a1aced679178dc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/5db0ca215a817842ac911424a22bd93fca130905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/60fab71ea695185b5318faa5b9ee75709b0571d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/94f3ef974b42cae30b383d82ae7b4b23b0b0acd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/b890dc1d9dbcda1238b49a5c037905fb254dc321.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/fb7deaa017d864a3628e0c6d1fdfe74c46260141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/23723e855ea3c38761ee775f08eca0fa8e4d8483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/5ec57abac3827ea5920a122a698be0a55a1cf376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/805c33a3e2845191195e9dbbe4b65d04e0ded8c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/93249a451e8e4fa5fc368fb5156e0872de5e7648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/9feba4ffdfb21905e352d4c6dad52626f4c66dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/bc041449c5bf0daebd77259727a926c6279bdd62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/c40015a5cc9f132951a5f49a01e603739c0a8725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/d0909d883da84309e403c9994fb417e349993eaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/ed52fb7ffd3000647491ff2972b18b6d7ba80584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/0d4fce2306404383249184fd78217ccce664f0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/1571ea5ce81d7c354cf6b4b35ac0243d4604ab82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/4e6189c40890ec1656ceba8c0ab7b98126809951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/7ac993f964ec5c0fa0f2aab74ee2f7da1fdf525e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/95e70b1d825e476a634e8a08c5b1a366d1fac9cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/e0f545b90260820397eb9fc06ee4a75f61a9dbd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/e4cb086fa050d3bf997730f696916a9256fa27a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/e6cc3c961c927e89e3e559367d2383708cd1f9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/edd0a7f599cd148457744bce1f6c4031d51c6c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/2dfa58c1309b0ed0971a27ec8f9e6805c5ada1c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/3489bd254fdd7c3bc880cba6a92c46b2742e86c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/49c5485dad2addc068139168d62724939862578c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/91c1b35bda2b089b7969111f44ee53d207169a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/9f5e24607c674acbbf83801888eee5374e7be03a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/af0cb9bbcbcf96e8946e7869969e1fd35a0a198b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/c2fa4bdcf381acc5714ff4aa03f8cf21885b7929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/d9298ccab355435bb7bc8c684767ad72a18e3cb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/f836de175b31f8c963cf3fdfd76a1360a25bcfcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/06e7be8a9067a599a4f3e867b9ff40f0e466a49b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/446224cd502fd83cf1744e9380c11246508eee47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/63a56d267091f63d0f9c65d42cbe2c6d37c2c775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/d8479f4a1e3c59692a637d610d3350f09b3fa2af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/ea7ee9052941ec4ed730c769cbf8c530b8c0a980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/ebfce2bd2fc14cbd31ef6fe4a3ca30e00148af57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/f7c061ce3fc5a31ba355e10314919fc0fc68c8c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/f8400ae52d28ba23d4f04cdfe415a50a88450de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/f883099625e66b21eb79f6e2631420fd3bc01a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/1a749b28089339800ff5ec56bb15c744716e673c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/1bce590d1acfdfe95643602dc0630559e7249c1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/379e1ee4e0361199549becc9c99975897c2c7287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/40b7468b8369d891757008724b742dce24318d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/454b83988c757c85a086172753ae8a276ce68b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/6b2ea2a9293c7bcef88233ebdf496ef4a3f46192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/c6fef6a9f1a4d7420b71f9e3442bab515e1fa442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/ed9638f7bd8b73042d3554ccd505105f359492b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/f268291b44d7f3b97aa15b8b07f038fb1420d423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/1a2b0822a9d7e636dbb8cc6bc5956329a8f95998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/3edffca512bf693a40e3a8d6e78bea73789c1236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/7cfb76119e63715da23b2732b7c85c92ec689eff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/966f07b8a8973b22fdc711af89c3e65245f80122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/981e5f3ca5c3a84c44351709b05b0efd28cc0d2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/c596987ecbc56bde944832730dc58a52963ff22e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/d3743ff423404767ca3df713cabee6487753d3fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/d6ae035e0aff2b415878de2281d585b4e7cf905c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/f292b395b568464495df6bf656121b7847e55dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/0697588fc50cb5ee4995cc3429a332f910bdaa30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/4b9b7b22c958f246777329b4bfb86f31f08a555d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/4de6afacefec9f6512f55c4544d7740cac3647db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/65dca1e2a15bef6c618a3659ed3c842d4904f249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/7e0c6539e0993cc75c8d759374ec02f836d2cd8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/a3083130bc7d2077b88bd0e1d23a63f06ff71c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/b3b0a32538f58776a9256f92edc14d3069da3319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/d4a8338920f9001bbfd84eec4d88b2089a9c7491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/fa73faebb571492897834bb2a07dd85af3dee598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/304f4b2ccf0b918308ab39a84a6a5c5f388a0c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/3e594861191a38039715799a84acc87add56222d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/531da3f48ff6d4600ea09cb3d485c097cd2377c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/81104d71c3159b5db0d7b208c1ae3e23450243c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/91af8ae9bbd2ebdfa7ed31280f06881a76c8f0e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/970609a3dcd7736a87872ef9da0cdae065e87fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/995ee1b9ab373e38f003fab296380550dc3e2eaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/be3b6ca3dbb3364ee2b7179cd2f0d9f5507cdd1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/c3f15de69cfceec278cddd7853bfd22e50134b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/10565e8b43b7386dc83e67516da1d1e2700e557e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/40930c0f483deb4e693252b11de3ec0ef460a848.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/559269017ace08de245fab28f2d4f7cdab36ff85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/7b3cea8384dac35b08bb9b17c2b17eb365e2104c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/84582da1c7f681e6bf4c969beb789ca8f9c583c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/9fc375d0bb32b537693b780fef890d5851780959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/c1a9bcad96fd9745d4b8214a0298844c2b536d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/d2dc4bcba34bd836ea330460353de4ead0a350a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/f8b1431ba1ebde56ad2757fecab9adad3103dba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/45384d0b2afdeeb4dffb3745d9b4743e63ad2e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/5c5d4e118aba8557fc9e68b1be55f61311208671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/85238ce11d3b53f82e1d29bf7cbb8a2c72bbd908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/a4dc99813c57a4ba92497a020f25457f37b55c69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/aec9ef58dfdb9bccab0aa3d80e9b83bdf0b82248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/c14cbecba3d8e3d1b4fe440ed1f68e594b8c47cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/d093efd1728fb7ee227a9ecddbf560681cdce6e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/f0792f9e7cfe776c58a9a3815851f49a5f983abf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/f19712d05b9a8e67e2a20befa7a7814f3a668acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/2c64b9d54947cba42639853a5dcc01826df9334b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/3a2c5d34e5cbd8e75924baf297a4093eb1a813b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/46349b7fd8869306c7c2afd89670025271230e63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/4e34feb7044ad6b598cf1975a783e1154b31d3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/5cf32de8ece19cb61281c665f989428c6437abdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/6b09db704d2dc3bc804d58f017f8bcbc32c37034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/94732ff94f0513454b5b4013f1f53aa5c5983f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/a42ead2f3382c5fa076d9638a5934bca106bb17f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/e12c26e90306c7df4956c5858db57bd1687e610a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/046a43bd03e9f9e82e7bee9860826bb80ba7e419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/29fa9fa4f2dfff7c764198a4161482c73e6d501f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/3ddc2ab3260a7b471420dc15825d6cb54a772e88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/578fe051786349f23e46f2e19df635953157b8da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/7667c751c8459f933aa60bc7e8f65d9c388afcac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/b96b66524a2aa7ac9570829fe938ecb41159355c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/bd7e787f28c56553f53ebcbbf1876e57f0cb8df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/cab01e19a22ac2b1b354105c4551e1b3edf0aedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/cb2135b10ce868814c1ee0e23fd356a70252c055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/16c383bfd8c4faf54259a31ad4ec5d5c5cdbe0f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/2618e9454f5fddd3fca9ac5d736b753bc24cdfc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/35d280c54e6daef1cb348f562137f223de2da1c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/7b724ea01671be8bceb39e2c0240daf779f5ce84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/8430fd3f70763f5c88e84763e9bdbf4881881c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/b19547b01f6a1dac5390cfbddb0fef174cca24b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/bb0aa72e883897a11d58faf866f28f4a5290a9d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/f4afd36613fc1c055336c1461c719d56dae4aea7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/fbb97926977df725789507427c9f3629c6f9eae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/2dd71bee6c44e3ba5b2c675c13e0dbdc33ca598e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/526e5f72d0fa0298d6c5eb09df176b275f6f31c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/5f8112760e0f0909f04ad5d76a439e2864f7aa53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/6336101da9976749b67e345b3e04ebb2ebccf5c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/6e2fc2e17e99e22ee50e2d059522af669f4562a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/82a7aa193b4bc1f8d0133cc2c30ad95234c0c34f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/c3f40ab4753a275e19512d9ae3a533ddce27da2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/e1f060e04ce195e8ecb072defd08294d70b0d5c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/f9523dcc5497fc10df426f4d4f03a5408fa4149c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/32d995ed1467318670ae824391180520905bce81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/3ca9dee12115e938361841691783d96f6b14814c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/427034da886f020bca782de72eaba1ae71691537.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/55dd0a22f91d54abbe7da3236b37702290bdf4ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/7d903fa83c00492b72d4316b01e021ecc33b6c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/88a1ae9f6eb5bd91568111a2e69088579e85fb2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/c1d3e2677688ec3f10b48126a2d50bbfd3140b6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/ef4df195dff436414ab7392251205fba65d7901f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/f6b8e7fd219d5cd161a01097b0894a9949b16432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/143430f6d0b86612aa8d9fa7588ab27519e23b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/1735bf940242edfe1601e27d1e1c68c1bef95a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/1ae5459b971170bc1c4c19e0fc84ddb826a6c6cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/22af3dfa9f3e78dee42ab656df3d20239b5442b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/37de5dd937651fc44795d588e8a55cba625366a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/5ea68a2da4aaa0c891f0a77c83ac708f42540e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/921069b162c1cc94ec4bb2fd29a9d016eb0435de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/afb2a8f845190382a13b6ca6f9dca867c8e6f7b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/cf780be070c4c699d3af17d61a4e5c3c83f9a859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/1ee38f29125ddbea9e8ea8e828a4e6b565ccd5a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/3e130c6f233d0b600097a4913a276f5a105b426b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/4d0f2385f5c61e72d8260578220a435cbd2ae99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/4e648a11cc9ab7de05e5c62c130c62d1a5aeb141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/59af80b85bb819eee3eba131bbabceac682aa9a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/6cbf240da853d5cd892ee8abf9df0e929cb51b0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/8bbf5607ed408dd91ec984a6fa522323c8f83e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/9151afed55cf0601be03b1585fccb596c513de0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/a93f73513bd27727f4f056c5432a1a097851d786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/1ba62e517ba3df9a24472de49cda93cf1d15564a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/260e2ccf261c431b9eccbee56ef8a9a05601b68a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/2e46bee964d6d4e472c1922a673a1fb878f5bb2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/7293d1f0fa0cad90879b038ab02f8849eaebb698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/7df3757c72844621aad50caddf8f926e67a49725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/82d69e6187a42cc8695120263cf3e6ca81ac8ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/844bf85a3e9223f2b3fd6ddc55ba4514bd6d6dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/97e10c40c24f9f432c34e5801df8c754ea5073c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/d8bd29d5196808dbbe11e6b91ba4e58b1b6a1558.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/1794a027e1f00998c3a499e263d43c6e70b60a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/199454274f67f57f7df3474deeb7ce62ebd69020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/28990b0cb0158795901694a40aa47ed2cbb245a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/587470d9bd09c910f5b669d8e21f42865259e35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/6e862ceaeac46bb25b48c837415c7744da31fedf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/910eef746682b5880ce54b21a2ec544dfc926854.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/bc2ff5f0ae3609d8dab9d6d1d5d73e9a818daa05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/c4594d4a978290cf7c38fa7064f082ea81074230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/fd51bc0ab2739d3fa4840885b5aa6585f65c5ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/63527d109b2889bc609f083802e314556f485555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/6583e96edb7c161aef121d3e747c5c230959d355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/835a7fd64fd427366e5d8e23e476f44e198ea108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/8ed0891a0232c9b445872beb70a2a0b7c330c1c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/9ffe8f4d3a69b3f825ce8c42c886d0c73b849c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/ca6221d4e6aa04b6bdc82cb6bad611d64fc4632e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/e04079720c40196e251de8454e0d69ea46afacf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/e615f98a0440ab066902fd3a30779ac96e1c59b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/e9166251994146ef17fde5aa3a431b3a5a0e8521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/1739e77bfa301f2a122b52e9e85bc825de2e31a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/470607b3c8ad50ad77a4a56f1c4d24fccd36f4ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/53971561e7aa70fd1894ebb7d51eadfb1760bce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/6c70983700a936ce53fb9962a3b7b773b849fbe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/78c1eac73bda72282688a4138d50185915658512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/ab01165c20dbf5acc415db1aa67dc0e1487cb9bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/ba2c6230f9a077230501600f74e2d6c0117e8fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/f187b5161a344ae973352fd75951f319a2202cf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/f3b0f59ea88a1761c823e5645cc93f045ec086d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/05c0434b987ce78d22ad2355d84184044172bcba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/19495d1aff19bcdc513fbc59b67a7f283999c212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/42855487bf79cd5f0b4ef71f094365fa9f0dd5c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/61bc2b2d2da669713992b3ae296bc57170836fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/66bf968dd3b34a6822f62548eb5c09f2bde98060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/7a0e8c547b1835fa80da9f39b5be854115580a89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/d250d7cc011999f70144e4d51e94ed238a8749f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/dc60fd4cdb17f2f8a493b927a1c49c0ff9445699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/eff980f84709331429e8da6d8bc3dc6117bc4cdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/5a88f02e134506a5d8f98d6e76b14105eb3b5395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/86bee1cafed92f400d5ded9587b9a3fd00b8fc77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/8d2051c75cbf22fa5c60cf0ab2b9c43fbb2f61e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/8da614ed7a5796175961447a06fab7aa071595d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/ace808f05253735b39e7266090d73e886b748113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/ae18ffd81e99924bd6f264dde369f98199dbc3b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/b3fcfff0def9653e20bb195fc00aa6e11bbfe8ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/c03fa1d98638fa3d9bd841f615ad01743dc42aec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/e8113cb6d64bd636ab1bff6db01d76d48a28fbf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/2708e37d79004401a69d66645497c3d1bf16e384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/38d8d9443af0f83f82fa724ba654a43453f97f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/3bfcc343bf19c45df6453d1af408cc90c3dd1022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/a1368d4f8b41b4803f13ac3701fa13e9842deb8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/a171ec97c02eba00c5f14891819f330e0ee4f462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/af552da9e5e4ef8dace40e3e646f33157dae17a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/c743b9399c443bb94095716132bd9a73ce1ded9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/eb49c976aadfa3010bedc269e9a53361d36e1b6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/fc4a7e03c444a16e7e53733d5b7c287055260296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/010fd461f6d1d31b968b46378796b513c747fcde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/06db2958ea33105e69a6adb84c948781a24e4e8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/13e07fea56f52dc7fa9456dc2766d7395c6a9302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/504c1325152a4cbaaca09f3173349994c0207091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/8317a06e762290b5a6cea3065d0967fef27c9bb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/a33d3eb11e7047162649fc806354fe8a3cc390e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/a523f4a6c73fa9e36810311b752d72ce041e7407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/b5c951b7c3ef87f477dbadda7c4d7f7327ac7a11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/e7b789a318324606c948a656d94e4601e81935b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/2633583484c5fd5cb1a6db32765fc2b3513021d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/4482b4765d90cff7487a9a64aa0a13af091b7ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/4699e2f8f9ee813d24c3eb35cedc724399811e75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/792a2dd98aea5735a73ed433d7808d19de5c3e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/7d74bc06bdd03dcb17991b6b95f16098bc4dea5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/8c73cffc3dc3b92374804a90356de78957d8c1d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/ab7098fb109329f2428937f1f381de706d22ef8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/afa8cfc7582b67c8f656677e6b9070b85b75e716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/b00ab69b13ec6b0a1cee07f5a54f608416553e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/05c295acff7c48750f201f667fc165202626b75b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/128e0f7ac08dbe0d38c57c33d53e58fdfbe18250.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/17c4f6ef5131e6b43735f0e83e3fde502b7fe15c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/188daba42ec7a730dde816a6983c91351c9db5d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/1c5896651ae3d9b1f0bcd8e05d4a470a880a42d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/3221d4ed5268633c5f14a3df7cc3934491c89a8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/396949856c92e9d20735fefa413aa60725d63972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/c08af2a11685622063eda13243e00e1bf41f1c9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/fd69145a71c7572524a68c1c9a14c0c4cd30b504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/1a5fc46db419516a1b146ed2b7815d8a6b98946a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/7e8455eca2694ae2724f258cfa20d0ff1771d98c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/a0dcdceef31380e03a19ab2fc5d0ef8d8e3b588b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/acc9bf18fe309f19ec601906a8d62ee7eb51856a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/b2d0f298b9fb317d2b2d207499ec72a71a267d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/c65ac0be58c2ec7cae2874d36794fee28939f472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/ce7c87fd9c96dd84132aac3724c56e40070096a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/d0dceab18e7416a5c3be13a426d220612290ff5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/e51bcfd5a31fde11bf91c04e53823db206408121.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2024-10/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2024-10/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2024-10/output diff --git a/configs/cccc/dedupe-url/CC-MAIN-2024-18.yaml b/configs/cccc/dedupe-url/CC-MAIN-2024-18.yaml new file mode 100644 index 00000000..179d813f --- /dev/null +++ b/configs/cccc/dedupe-url/CC-MAIN-2024-18.yaml @@ -0,0 +1,921 @@ + +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/0bf9638f4b669db03aec903886877cbb7855332a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/1b2735cedc42e27b00d246200a39426a9ff1031a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/3f546d6e5abfeaff58afaec4e4e1fcd34857d136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/5298d73cb75586c0a61daa41ad28ef901cd474b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/8731f7d9a103a87c547dac087eb48cf12331afed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/90dafcbb6ca464c09236a4d46e6a5d17a9802249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/9434b3569c3af3e3dc072c92c5b1e6d070365a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/bf04fea003b56cb3d0d192a260bb95d22a268459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/efa549d9e990f80d3d09e6009436750b3ba3fcef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/35a68a6ff09ad5d3093d83632ba38dca0a8facb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/3da828c13d959a6f7b723c776ab3985cf3022df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/61a2c91760892865d010479001ab122609bc6ee6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/81446e0eb751cf4e41983272411791f218aa6705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/a033fbe0d99ef58f8840dd31d885a780633d28de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/ea841001e27b0bb07ba1633007a72d0692364fd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/ec47c83ca5b6c388f796a742330dee9329173b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/fdad967c243448c02c63b683be1d8ba24e914864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/ff78175283838b301dc5525c6493ff121c581873.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/01add4422b4e4ef13626e8bcebdce3341ee17c80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/192a08f8d2f8aabad80fa462548bd85a3b749843.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/2d1ece5ffdaedb716debec3ab1833b37538b6f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/4ccdf5008c2105c76d9c7bf4df9a5830f2845971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/79a5892000302fe95b671afa1a6994cd7c7c3a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/8327719fca766aeb22cb092d409d4022d8e23fb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/97105fae66b945dbf6562600e678775df46adb5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/b2a358b4f21f8f9c4752fc6805223c92e9c6c525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/e67263eb7efac9c1c66fc10f6754b90bfd57523b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/010527c8640f2fc363f6a6dce4b15a1c496983ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/5c4bc746c740921a21f749b973d1f8a9f9a00933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/6218ac543a135fdc6bba2569fe0a18169a20fcbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/9414c80cfce63d87f88e887ac734904cd6ce63af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/968e2631b2df64fb829db943bff575932859cbf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/aad676b86e211aab6c087052f0e2df8ee5a9107a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/d48621ee317a83c9c98a54edaa4d4e3c4bd563a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/ddf414269f5e8c1e18beba4ed8468851617f87b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/f3c58649ba92782f0144dd4f00f92a88d5661c6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/00a1d6831a9ad2b94efbc4a9213c97baf42552ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/21dd7da0ffa531d77ea97dcdae106f49f6e02ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/4237df16c43ad8ad42461dad7722d7e09d1ae8f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/9768ae115b0d04f117e38b956f623bbc6637fe88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/9be98401afc60e2cc5c166c9694fc6c9b0868f39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/c365ab5fdedf2df2fc46e0e8f9627a4b67e2a461.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/dcece95d7e39cb5c730edf45679747961dd89429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/e86995cb879aa7c778a36ec1caf66dcec4663304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/eef725d0a1f899f0ce253da26748996c068c12a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/25f46f19a01c0085c781bc3ebb5b74cf7025135e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/45d2fd57b264c6800b10d6fe45d3afe4351d7730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/5bcad67bd4400238e19379ca08b34c039daa1815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/5c1adc2b35945dbbd938433fa1f6763f8d74c18f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/6cf4deac97266b3b504efa5b227a7a99bf067d3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/875cc88cb889830ebe2a43e97816d6a6faf857a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/bb35896e59892c8939bd94dd944eb50dc95d02dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/bc9da75392c52da33a26b99b303ee20472335b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/c9a664f07cfd886d8e5033df29d7b05da1371df3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/1c73070b6ca4ff7680998747077724711ab7a7eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/4c75c57fb3644eb489d31b306e06be7dae0aa164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/519e6acc9da83415994169a0454795e28699c78e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/710f878e683b982528a33037472faf57f824788f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/a442b81ed1cec232852873df648007782cf749ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/a668574590862270a66f25e53022e063f1461b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/dcd26ecf9baee63d40cefea6c33d6f623163f5e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/e4f010770d26a05807923b9cf27fad53152f73c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/f43bc4b9dfb38bf8d766fce8564950447d39b87e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/1d82a6a701104eeb3043a822cfd80271e4895c14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/217c5c75a550979ca17392dde9213389bb8ad1b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/24b640c55079d0373630837a7e61cf727a304099.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/25bf4ea19f39b016ece2bdbe213c24115f027a91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/4bfa10be979b1db6979e3f412a81f5f6af78dd3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/5c0dc489bcdc8a0b52d0f156c7b9e261d9cfb6a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/a25d677c03ff8df414f6759fc95017ffc4575b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/bf2c40526a57d3b5f0ed32b83392ce02649ef454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/e269255e85301cd8e86620feaa6f5fd98d74285a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/0271b18eca21e94b3766973cf2c7b8cc5190c3ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/0bd8cde29e490eba667d72450bd5737fe99e9b0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/7277e16bc709434c80f6fbb000f77fc000c4c194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/749d3fc4f24a5f93ecedb869de697a36566354eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/79aefee59d97a7e8d67d76ae626377345a2ec231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/7e0165d18ad130024ab98f195e3f922d612a4f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/90fb97d3455a0820162be1df6517841089b40383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/a14c25a8be490173531822e4998b3c6feed68c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/a8245dbd1537b94fee41e60c7fcb1a65312e37b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/2986788f4bd0d4195090343b4ece6bbbb2a67e99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/3886869e5dc888e79710f7e4480cfe091257a41c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/3c6bf9984b5425341dcfdf8571b1230a0c04f97f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/5ab9b523a3e6c1b648e9d2fa5b62ba71de4bcda6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/6a3df68c0f0741787a97e9650c1d1a4fc8fb5342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/8b54208edafcb87386c8a25f9e796f2d65a6f51c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/b0d26f16e3b8f6f55cb3b2046f09bb40c39e0fd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/c072b5f9c54312557de65215f1efb05247a696d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/ddb6d63bdb1813e9c19dd5fae168d5d582b47c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/19340d4805b2368ce81db03242f10015c8c827f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/20a5b7e8e78ffe379a6c1c3b9092066d0fd33226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/3025987dab50839fd149fd667ac12e22fc1a25de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/48706f40dc2e6f22f89de5f76ec6ca0b4edc8e14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/517327b4c222dd7a38179a84ffe1371969c4f754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/7dfd8a008ba91d0aa0d299231fed6924a3aa98cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/888c5babbcd4d0051af01053a40f0156954ab7af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/f0675ab8342501e0daba2aeb561f62fc91c1af1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/fc9ad03f4fe593a66d7ddffba0c2cebe55e7a628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/0162bce352471b09fa5a001d29a56098c99d17dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/29ba9f097b813a7c8e673227167aea51663f69b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/44e92542f049597dc9ce528654aecf6dbe917f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/74100051fdbed0d28408cf7debfc76d1b52ad34d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/b65e8d173504bcae54bef5a9888ed43a8416b7bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/baae15d55574e6e865451bce4792a1d18dc4fefc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/caeafeecfc07be87ff0aa53e26e12c7600681cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/cc6f6df0af38165c441b5685e72ae5f76b9c1374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/f6b6e813a1e885779d2407dd2ecd050c238f36b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/14df53feca5d666dce16771d8764070de01f918b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/4b927a48f74e994b85e859e8381df28d8862419b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/4dea61f70c1d7b2c0cc3f494a514096abb39cee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/60ceb0ec5feff15992bf89ad1383c8118add11de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/631b24823300ee526d8304f4feb7fd817237b516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/6f8283c5dacfff42f8047da72c1aaca6d9243e3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/bcf36bc77ebabdfac813b087d0ead382bff81b2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/bffa93fe6e3b72a370c0b47616c210db12f7c588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/e733614c1e1ff477d9347d09047693eccf08400c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/0a5d3ab8825d11f509c4d18f6638ca19dd1c674e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/24a30a62508caa699f0c13750d17f95d01252840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/43511692d4b9826d6cb2363411d7c49faca88d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/9ba92e614d4f3bc80299457b8f142b63780ee526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/a2f60d6cfee80d31c7f9e4a0cfa949af185fd1ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/ce4e0b385980f812dd6d48665926c61f6c1df4ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/d6342d4c3e637cb97f8252fdce22e144eaf9f903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/d9a8a5c62452419607d749a620f4dc3f821686ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/e6ab411f861f2adc33db0e7a6ebef5936cfc00cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/0a984796a0ce103e28cb198981af4a4b55e14305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/0ad44aef8a4341dccf1ea8c071eecf69cf990042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/38b05f7e353d291c637d8ec6085148d7643f338a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/4a6952f99833ca52507c86b4a1a4fdc4cb11afd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/688b2c1bc96da40c7f0738b8020d4c76a9f2ee7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/8e44301758f17f5e42f8b210b8f3664d57763c54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/b620706a0f39eb5d1491ff4b857a8d9434d76e5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/ca6244cd655c9f202388b3047b229d08640edd61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/e47fd6dc906172b090c8c5289b85d0c27b19ef69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/151f2563243576397afea6940a8b0af1f404bb27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/2876a4d85463401465397c88a93dbfd4c40f0645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/3d52114dc53407ec9a8d10393b406308f74a1348.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/3ff09a34fd1379e4944b64948b0086fbd67e66ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/784cfd325f8d524a966b6daf301752e4da5d4fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/82031fb831099a776d21bcbffbbcc032724eab2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/9ecc52496c43307380830f05e6362efb315c23f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/a28c01aece993e4228af4bfaf4d710d5a06c0e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/d8d45b929a26400a1ee0cd80eb6c247af90a1ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/01ed9944c88d70fccf0e8a106c4fdd1641a73d1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/27059d505febc2a53d198a1439afd6d3c9536a51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/27e8df37ede8aec15bf65c9ecc887e79ae291f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/299f59374a88a390b66e830f82b4baf818fb594e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/6c705805240538d5d5277b1950a6a2528acd8b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/7768fe17e5d66dfffc89ae6278c2612f018a19c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/c2a303e92175da5d33ef85c9f6f66886ba6e2292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/d1b932108f62b734a99c1a3c2e0ebd623f6380da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/e48006ca5e62be8d121c9d52f6add9646e571a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/097f0c7098d85ca0ac7777df04d71d6524b552d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/119a67e15b653b465f78cca537579cda690eed4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/3c05cb05641bf18d9e7478e47b81db7a1a651b3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/9f39c49999323a582e46c65096de314500e2fe93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/acb42a40ba4dc9660e803db70b347b584c00126a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/c256a76affbf25535732f636d2611ffc4bd46429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/da60d2fd4af42242463c0dab6e96b7d5e6801192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/ef24a11f077eeb89e585f9bb349efba1723b2000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/fb7fc1d674c56e9685be838281230a9b854e5cfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/054ea892b343b29ce6b9f0cb3e88d0dd283dca90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/0c3e65619b2f930ab2989a5ddd79f73940c0c803.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/25c287eaf9b6f62ec7424be16242fbd9a66c57fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/3a3149247cf1abdca1ea498e08ed38cfa257358a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/a1a59f5b07c56adc6640600b3e7ab8a5e942e8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/a6244fe07d61132776a14d2c9d24abb5248627a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/e8633e4a3e2bba671f716e89bdbde40124c2f95a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/eeba257a0eb9294a58616f3b7c85fb756a8278c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/fa184ee4314a91fd882cd8cd96c81f0154fdeb30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/07ac310257cb684396aa462eb079f6344c0ad13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/3d6c12fc463f958142c3b104ed074060035a59c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/446ffaf8effb7507b8d63bca1536c65830d03584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/4c03709ceee24b646e6ddc0f56b38c5c2118fa16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/7b39aac403ccad6eece94929c05240f94aee3d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/b8cb81bfcbf8674321f047042fef024b1246fd4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/cc828861f3328aa93650e12abbe96a1908ed4330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/d4859db5e165b87fdbf6c54172ec035fb07d16c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/dcbd10e9598e7c2d57fc770009e0cd3c350705cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/1ce36fbe9a33481110e65c8f23772b424b724875.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/2996a27161e42a8c351c786857a8ab49de10e232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/371660591173298f4c25ef839db03f7173c5216e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/4e078ab7b6dffcec1702c12516af9d7ee4d01a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/6b1b06e28a19fcf974cc27828d80be15fbf2b4a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/a52aef65b5771eb0be44148e51d86a4698a10d5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/af30f74c3a5967e7c09f822352362a9385803c36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/f370f78e08065cb47b6e73d0688395c35e419af0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/ff61059dcc686b1bdcfa7f6339f90a09f4bc9420.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/32f0259f568c40d15f134f918d9a4f888c04dc9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/8806303a85f4d94d315325f6003f9afc41efbb2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/88ac1f0c571dc2046c51e920f171e8304b981758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/90d5a27af6980b0bcdb0b240ebd832d17d2fd693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/97dde0bc1910933f98809e3cc7e53ef4836b5cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/a857afc568bae8310a2525f0f582cb5db5040e39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/a9490a7c05754435c1c7ffc2b1b8928a7cec6059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/aa130df2b679693c9933c1db0f37c32cb442d7cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/cd4adcccf5583c01237240554ead86701fca9fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/22c1f321c14886ad3379749efa4b803a32d1e285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/2bb8e2a1f3eefb6868e7f16a039c1024bce3dd27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/39b21ebcda9b90439b0aed3528608264c036ae72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/4b21de647cb8020f794ec9d29c38d3cad536b9a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/90f97abd4d7a16d6778519c4c492123c0ee61fc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/94d609896d313cd1cdc70a79cd34e9129a0ec7de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/d880283649d0e5d0360e582445bcfb0c73ea5895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/dc124fc281f604ab47a438bf4ea4d812d4866d96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/f22cedc208c57195f093d1f33afe9e1e46274fc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/0b3ec5d865692449a959b6defc1344daadf86f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/2d091d49a0e7dc1d5220504ecc0045462742f36d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/2ef7aebad6097d2f565da7e06d0e690aacd42feb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/34c52f296311af80be60405a8177c9ff52715376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/842c05e0973473fc6f694d6d2de3e2ac1ff486c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/8fe47f069659518008f7c76d9379e23040924662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/bd21f80272ab38a3b0cb6b89a7d543177e5b9544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/ccf84d16f1610b81d9d3872deac587abef97ddeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/e9b6e285f6f8dfd5c8885ffb74d28e3feb476b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/12d70128ad4a3ab71d2ad27808c756a63dfa3e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/38f77f699d7364201476e0f8d22c64307c75b85d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/5ad6252a89d795cc105f1829c119e0eec5dca9a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/bae8b3f199d3b90a43e99358a748bc95f2ebc6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/dc5ec6e1772d20cfbf40a805c6138976b3b33f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/eb0a14faebd0733b6d032be5f9e617ffec52d35d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/ed03581f97752e1ea974e474ecee609f665759a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/f0113127cd3b30c951f719d0f8ac43a2ac4ba3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/f3b307d8752d7503311f41ae610a797abf2c5062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/05e745220c50ac52f578e958fb4b703a63cdbae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/35fbe8efd8cacdf596a104573f02636ce48d8c9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/5411472ceadae080aab6b1715f2f3bfa87cef857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/6f3b125e0df312e77b06b620c7f0d15fb3e81002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/7abaeaaf6da635335036af208c45e7ab7ff6e3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/85a53a1413e54d5af5d9a08043cb7d09e980917e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/c0270aa18e6306d4235a707b0d414372bba18ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/e89ef899653ffa205a48670735b28929064f2aa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/f1586dfba29318c09e12ab90f31ebf0541c0fe7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/0475a61f956bea8d546dd3bc20674650d1128db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/36ec1576787e9fa53933ac7d1c72b5aadef01a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/390bb6bef904604b97b9d02f1d1ab96817fb644e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/478c5be286d9b6ed7e1e1e1585e9210f156ecfba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/4d9c53e64fd87fc65d5c9691aa7ccba1033db18f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/5ad2a79ec48d29e95ca34bf23586e81c44b5bd4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/a5dace1c515e91a002d6d8d7f71ad6213a651fc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/a853122f0353a65ccaddb3f562131e2b9ad76f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/da8a2af3224e8f5c3fdf7053c004856b4d8288bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/1012a7d1dfb270be25fdbb86eb5c01ef1c8f4e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/1f80980580723ede911f0adcf8ad327cd38c7b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/2e9616c776d1a0a22eee978f98ffa9f02f45ebb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/4b7e2ce83c71ebe4064b8b893d3da0242ea153da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/4cac1909727cd7c571dcd42af50cba782f5bfc9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/5328cba2f24bde54d417bfb050d598c1dee3a961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/71b891213cce4befcc18aa547f200bcfee13f050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/8d4a822d71985feb28e356a0864c91a8b7c22ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/e1d86103016e37d0b9c8b13f2d2c73964584486b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/0093d29381f7a1f1af6a8cc354eba79fd6834128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/3628f4fa0432ded8adf62408f1f687b572f9e8a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/406780dc38c88976bd4d206523e2ea151ae4d29d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/59b09d93491e91160790837391539b3ad7a0738c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/7e93c85edba1d7b89926ecc50abd6ab94455fe06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/8c919fdf39cdbac827f8ff5fd8071a20c8026e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/b276ef3774f0309c1969bb4643ecf6ddfa319bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/f732ba02fdb4979d54a2f1f618ae3361a609a746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/f8c632c9b89a7bd202fd878d501d8656bba57a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/10affa9db61417c6c651776129da815ce2a36dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/2971f9738abfd7265cf3a3547a8304818bc2fe9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/51fc935ebbe9530d8852e1bc42df1156ec80f8bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/616c532f2d89e9f8ca682e3c08d5b12812e50f51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/828b7b01da792c7a3cb39b5ecad17efc24a93b6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/837f328f8fb6e69dfe4a3091baad04e2bf2f7ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/ead8df627d8002aa8c91f30b33813598df1db1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/ef10629e58248dc6b06431f59dfbdbf09fcf7567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/f1325b4afc35306b2941371037fd3bf1062b5f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/1e216a898a56a5e21663f0eacdd4b1b00f110316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/2a6cb39a4dfa97cfc1aaa9696672b320a771c45a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/3195360e12cac91011f278922946f7d07df0d3f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/5d128cb53cfd8d717e83aa4bd9441c09ba66a753.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/66b82f2ddd450daecb29a718c290a7b083ed3d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/716e891c5a11328b500bace3b1278966c6f2f329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/8c22dc34d6c9ac6a0446763065e881b82f6ee754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/de19c1174f285ee1b5bd94c285ea8f531f1bd5d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/fada2592fd0eb4dffa95eb5f64728e289fbbd998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/039742fcdc381f343f31e56991833c60a7148ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/1e93758cdd37a1f477315b4b13b1f1f0e62b9154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/7ba20a81ab559a860635149b53e6585a5d7c17ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/ab3c86864c650e0853b73c3852213af564061b53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/af3c624abeea0311f3aa09ba08ed34a932c10f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/d288a6605cf0f0ee517afe62a36785cbba13606c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/d2a7538e11c7a7bc05479059a16c9839a8a9b442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/d68134d0519bd9d8a47629fc385d8c28ddbd4d1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/fe00d219c1d07bfac6801888196127003c8dd1f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/1a8933b60de2b7516bc9d961874607969268960c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/258d161e4f889f47f1e85f30ceedcaa6612e747f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/3e5c7b20a7bf80cc36f2c68da03eaba2cabd640b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/4862d8d44e3a2061c7d23fad3f01e2f3d2b03978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/50dcaa4f08f9c398bbee73d69d71d6f2fb53e7ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/58cf536847beca160a58a1b643e0f46fb03d6890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/9ec7ce2deabab7dda4b725641d4413e4f9f378bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/a65f28611398e244f23a28d6cace69b4fe12c60d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/f531001780668085e826f10f887f042b121f1408.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/07e2705a2c13fed785531e55e01c03375ebcd3ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/1c787f6ed4c03814c40273ff2ab286d6972c5d5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/22679d690fe86bb555e32db64b9dbb8384d3ded3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/3c412efba24fa673caa0e6df36be59751edcf7ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/52bd379a00070824c4a6c9840695320f76be980e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/62a4bc99ffc587fad8a5d3f6b0368c2d2098f1b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/91bda3d18cc081eac2bf028f652585cf49802f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/9a4f9c32ae84d72a5ad4ac1122c0ca5a5b20477a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/de7e67960c1ceba6a68e1c6a7cf04b198b2ee1f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/2e1477e29bea71ab040f3cd4c89f67736e35bb26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/38ee811a959eb5b1c7c2f51df2ee5bb5ac04eaae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/4d29468f662cd6aaf79161c4be10cb57a04de3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/6885ed50350f90c9ffa751e92f9ebbfde2e88c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/77000e22f790c5408d806ac705b9b05afc64c40d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/90d1de21ffc52ddc8685141cd7f0c88c46b36edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/a533305e3f084f35cd06a2153564eb7b681d4dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/d64ec1ca1fb2305c044685ab030d4be8a53c0ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/daede23be8eccfeab000dfcceedf6a50c88009a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/0938981c6cb37a0097e8dd9e456ff77711462307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/2106ac25418a2077d2ec3f6d94c6b1c984dffe74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/2b79efd04d39697d2c2101c8acc3c4d610260b22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/9456ce57c3356fe2d9506b14eda166d8d448ba56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/b1cef4f8a454532c92ea57a5a3d071e66aaea301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/c7fee57e483b03ab3276c3508bb169ba4f3e18a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/c9188de20bcf7e014e2db3632df1a535b6ade5b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/e45cc81c69d81f8d08a998affcb13cb322dd031e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/f607337d3aaedd8df3a1844ed837a4e79a22cf58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/2d6d993568af428eafeb59157224cebe44a7cae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/35db600cc09ebbc19da9a751e308a13edf348f19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/3810fb6a0cb6ced64e9b00e63fe4cd71a3d96b51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/9f6b6f58aafe2095b944e7ad5b420d47a06ea28c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/aa3dec27a9bc2c4d0f8300239f968f96f45e2ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/aa84e6da5c1c9ff786a4aa16d1d0b97c380bf4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/cb2553c61b593182bd3a0160e91e664071bd45cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/cc96ea1af83327a7c54aee08fbca28e56c4985c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/d241574edbae7b989d2eb3fea33e67f0f158878d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/35f878553dae68365dac5b87f95866e03dfbfeae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/43b5efb88dd1eb2c437bb08393e45f89007793cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/49079248225d77f28050e80117a499c374b875a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/4caf897bdd7fca3e7370abd5383540e287a07a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/61fe8eaf0b49ecf01a6cf7a7b39618eea2aa51ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/65d23ef5a2881701fb79ea8755df2e733c4480aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/ac101282485a8f9c1b42538e5407f25997a3ff06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/daa119cb64d1c0b4ed92af47d2ad6ace040ad9c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/e3bee34248c1b79810e62b932e3ac8dc897b20c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/1661f9a02ee1f0df9f5688a250c5c2f50dd3c0d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/1b38c66aa7e31fe00ea9b315444c9fdbbae35bfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/508210e64c0deec9142f4ab92363c43bbd859bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/5e35b5465c828e8957ba2b556cdb2d96c73fe1f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/5ee31f14dad5b20502e6beabc21b2f27bc0995fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/64c272680133f113b4b1af0bb3a97cda8472eda1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/9bec187566730f4a95e50a10642b195bcd4e6abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/b1e124d8e759913416582187c5849f7b63b22c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/c5518a344f064a98c91d913feab7c9e153995024.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/1609cafcc54e43082d89c7ce3b3690051d5d7565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/33a8703901674158253f9c8c18637fa1e1b21c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/7238f521bbb7bfcbe14389c5e1b4c59dcac828a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/7efdaa8c9a1e2c9fa126b99eda5bd78626fba264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/8d8b10b9c565b54c36f4cf6f4d201f4c145739e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/968092629794a2e7248330b3cfb80303bef083eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/989bc84b356f50cb0f08376f5cd80302bafe5a8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/a0421cf8e3edca132a8253290aee915b32fe44f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/b882897f8a657baedb6dc99e00eef8aaa8f1b2ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/01bd60e0997bd5c4d31d68653c462259bec826a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/2f4e7566261f0d65db35e091f6d927a41966b7b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/37a9f07b18e7c56b533e20b9f82a9ecd58a5e7e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/48e70ae346fe02a5d6e10ec974e6bc7bf74e1447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/4a0d1d50d70dbcaa3076625532b6705751d7d137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/6782438ad0612c25ebe2d7a4cca632d39cbcd21d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/726b89a7b73ab5e49fa9adc217fcd3889786e70a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/9d4cfd6f3ec85c1d5cb7bfdc255974caac60e6c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/d32dd3c60c34f6252d7e8297ff9af0f6a10f58ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/000afaebc0218dce9935caa829de606325c8435b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/024d0ac3128b07bbeac9b33ef86e58c207783158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/0b98543f18903c8e2c241536ec2ae648b81438c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/272448b69d9f0bd3ad872ee817f60897000840ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/2b9f9aca926bd77ea5a90b296682c1e5fa871a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/55d572fe3c4359d30c21446db45b46c81a7209e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/c0be2d934822a8f4cc07f12e3972ad3344b67196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/c8254c60c22cf50fa55b3427f5fc814350fa7407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/e41223f1ea3567bb239f0c9b65d71646585b3c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/08f946301bd49a8ef130bbaea153f55e99ef45b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/09f0adaf73095b2382e0d9c57cf1635c54edc8fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/1642f7980aaf329a8b0e6c15254f570ec4c7e8f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/365ca5d909d823c9eb7667b38266b083b8ed6a49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/3a3d9ddbc94dd0dbc43364926c420931b57587d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/40fdd2bd99813c7869511b7d374b78d1fdc3db99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/b357082dfdd75da7e555341f110f9e1bb0c9c0c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/b68b7dae58a7a3cf31a8dfe6b39cdd44fa56fd63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/e40fe277e5507458226eeea9440eea044253ac84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/023295cb467402439eabbed293a26da94127515b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/1df40141608795fe7be0ab0f72a65a7b2034c6d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/22dde568bf2557e389e7fb37363a4b04a7fe5596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/3116daa5f03b69855713ed30a82ee982d4620f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/7aca8270e37f7804492327b603ca05071a471a8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/a68e00fc3ad457f9309744dfafac799de67ff357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/c6eb78cd50c6650226f92f7102f690fbd09a7a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/e60f0b10044cb08155fb7ffbac40ed1ee5c94f99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/ecc1ad1963e849f6667a8f3bb7641aee6d418842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/1d74d0871da6cb07bc8b1249bde1615d678a3f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/50e224a7d3803f73a9f4d6c8b52d307f6f25fa8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/53dcaa5fa2a8064add64c239234adc197021c71b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/74f22e983cd1e24b3586506c97a5e9b2b9f08296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/9668bc3cb0d126ca0b2bc9bbb7342e2154200dbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/a37854199cbfd84bd6e575fdfe63dcdf21344948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/acbd6061a40153fa9ec5adca9a51c682f60283ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/c54b0ea99f543a2258bc7dfbccc317bfeab82c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/d177ecf25ad50df735c880cfd58c159e02d74817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/1551417ec150f9d149f9aebd9f24c26ac87492a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/3542b7c1ebb1e2c3d3c923acd9686dd264ac6ae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/3e7a9089bb91f0a766c7be283ae11b7264702068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/4ebab9c262f4f080813f21988b536229a5024f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/5b25df2cf5883cc326ea11808006bd0b136c9ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/60d6d8fd59d866795484cc4071370dc3deb2d7cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/cfc545f9910687a94cca6d5a48cc4eb913f0b8dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/e184b86f08ab664ed8f7da21e28305168f80112e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/ea7b5bdeb17630abb6212859a5fdbad887393034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/0120f3a928fab3db38b13e3924cc795cc5fe973d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/4a670e454cc677cc30518cea0145814b9eae0b41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/4ca7dcee7707ff285a0a25ae412b4d0b9a410b9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/63bdc7615fa793b168f1fd00ce771a4e37db322a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/afaea7d18ac9b1ee3985510259d095e508b5cac3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/cbfded285d549b7a3b46dcda6276d18e65c426fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/dbbbd32e992fbaeb986de7567cc450cdff994e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/dd4263c6d4602b3733898d6fd640a300dafcea39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/e740f17a097b7f875f22a687ce600b0494fb5464.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/0f27508c328a45f5bb8f171a12e418f1552925f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/197da6c35984b9c457947ba358326efc21213e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/31419a9a8f049debf00c5ff31de1a1498a286986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/5e51471049f832e795c72b3a0fcfe37525b256d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/80ffbcd0581fb68c826a97494760971db9ea09b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/c61864414e8ffd5fedcb7aa9bdebbc95e620daf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/cac1bd8a3b5390845293ee2f0beacaf5a7412ed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/ec95a74b40c26b15196a46ebe97496da71d21f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/f6a51fbd9aabb93699fef4bb2a33e0911d1a9546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/02c779e17a0c8b5acf566e7b1b9c307c65b6f88a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/35dfb23d30323853b79d502cba30b9d3d9ccb539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/36682b4ba03e9d876c4f69b9a2a9815a6e111882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/575af4f52f8e08de45d8f8974edccac1db934ad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/8fe601232655d104de4b281d35fbe4a03b4cc6ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/bacbc8849815c845eb62fafb43f4b539db3eaf1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/daf627771147d6a6d933101986f928ed2dc5090f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/e7dca894df52b9f75253ed7b507da8ab3572b72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/f13c3f750fc39deb1f3714db5130f6de67d19b4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/1842957f9015f7294b5ade83d6d9e16076aec017.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/4783b06aa8bdb5a97f09fdfb5b6167aaa83d157d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/51dcf9b20d9da4d5d412cbc09f466a6692f0aaad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/8bcb792f7eb9af6144aad2d67443c7682df0facb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/a415cf5ef28e106b59c29a45c91993610866cb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/cf53978cea3f9152a63086bb40ed23dd613deda7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/d7ac8af8c4e9027e9106dd6bd48ab470e5c4d374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/db9ac04b998095ecf47a36852b0a28f5caa17bc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/dd7b9e913ef87737bb207cb71f7ad3e1037fd52a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/0b749df805de0f7051edbb3b6527b338374f3f79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/0e96a60db6f9aff75972b0a09ab07769056535e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/210d435d6f0a0e8678770bf402d4a6784cbe77c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/262defabb7651469bb80cae245c41268810ae1cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/63d68477ba910cd2c32768496a70cb9346c07051.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/a5d90ee10ac2fee3502118503eca6341412dff23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/d6dba32999765540f6a59fd0dbac09ae7a62ce52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/dc69ea7ab1a1c0556e9480cd0a11630ca40e1617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/e5c59c556d55e674411091d812fd50d5e1ed7003.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/0482295272a1dd8c313931543a241c14eac781f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/19c6ea4929406398deb53a737bfa01f3f8ae93d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/1d19719b7a59c5b9d00d122af111457695064914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/2762da58e384117f2cd888515e1f527057d8d1d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/65a148ef09552fae326ad2a6e2fddf4a4da0b3fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/7273efce2550c92751dcb09da9dc96f3b7f9f04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/8ab599b71232e4f6fc8d71c327866c728fa95dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/9b1bfc5b8e35d12df5e878a3c44bd1cc85fcda52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/c2421f34d4546bc3261f96f8c14b31365f378744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/35b3b2b77bd19ffbc41971fd59247cb744bee686.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/427b73bd81e8e3558e7b066ff4bac85df1481205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/59fcc90ad0cc01db6b8a5f3f3cc95df0eba58abf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/7d32ff843b8129f2dff5daceccc0531915fe61b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/ae6b7158baf04e196c9b0b3aec3b089f726613a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/c16fc5a7c4ca06b4e96f3ab9eb80928b3ce1b342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/c407b9835f76f946fd4ef3c8d557e5602c54779f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/ed72c3b19c223e4e030f60a4932b4d9b90eaf9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/f24ee6aedbf85e985c2fb75cbd4f96fda94213d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/02bc8e5a67fa969d2804464d57893692bffa9e96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/16f22f4693b671e5ffef1f70983c22fd8de643ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/76bcc4b3363934d66e216a6710da49486bdac620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/7ff7db72b7bf7c8172bd7d4059c85f1c4d56b620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/93749c0994ab6528f926e71fb0a183469db799f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/98a9ca645be0923350a709e6adbec965b281f736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/a579bca0106b5288543673bbf91b21167b3ff469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/aa884a7c416c531bd6f738f586a841f183640441.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/fba9673f7782e76cba945cba54b04aa8174e759d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/18f6c2679ea0f9c1eccd821192a9ae8ff0ad8dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/3e0f3b2ec8b538475f98534442361b42b4ee2e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/43476f5da9bdc518aedc6cd80be22f44c3cb65dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/50ffdfd9275249a1c16b95f77ff563bb5ea55324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/8dc00b8b9c556d210215e559409fa1909b210421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/c082a3c88748ca89ddea4a19bdf3d11862ec045a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/c8e92da7395afe3293bee7405316308f2cdbb8b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/d9ff2f886a58aab5bae8c75961fdc971010d46fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/dbff06faaee35f487afc37c07bab1eb9b052c65f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/02f66afa1c8f7e5553fd37a98a51513d09381bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/15d463c92b6eaed458190bd89172e5cda5754835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/6b6eaa34328aa4d9251ad2b199dd695595dc911a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/83157d64fe187a4b060cf3a3a3e128630e05708a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/a426804b08cdb85c757af8629d0d649201ae2a08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/a8e7138d8fa7f969c2a97e26c02a0592923210a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/b10347d7b0bd3043f32a829333f1f072765e02d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/d4ff57b55b8f41d64a56c721e2f5217142f861f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/e9c9fdf8ea2d8074edadb152ff1b1b649362f468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/4b40ec5cc7048462474b6a7cf175a84fa240aa27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/5429a9f11c77249b2808285f0855b4b31c13fed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/566ae05bd6a44edf80eff1c57b5eefac1805c1fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/814f92b4048b2d84c34d43a0b9566498bf8d7da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/86dc8574e1f3c709a6d0c34dcabaac4138bfffc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/be545e657eac28b2e74810cb06436db9abcce447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/e869be7161d492b9b17d0d48b9296c25dea4b24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/edb04b403e6c4a43f972f8b7c567ea1981463c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/f85c257e6fb1eea8ddad25a95132aaa1f6d91b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/08c8e34a34d698306a85ae41b7d593beed01cccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/0904f58dd6fb75867e9ddbdfb66be3229c00f097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/0d05fc12dbd9b26e2ec476e534512028b4d7c4c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/52082be600e49f9313ddb81929ea318fa2b8849c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/7580eea9865814c686b8fabb394b49503f4ca60d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/acda5cbff647d37d6eb81be84430a2ee7a6905c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/bf3a627bc6a6dda295a8dfcf6f3b1ebea46ef095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/c4df6706ac0a89ba9864c793e88cce6aa2ffa2e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/dbfe7614b96f983f50ae8f8fb269f3675e92cd9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/2041532fc65e993d0afd45481ed5a402eb7e94a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/21d7c718e39b359fc96813fb7a30bb3eea80c51f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/261f95a9ad6a9cab068a777d8c7db806f49e7944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/3b049a971bb2ddda8384a267efe902661ef35edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/40a37e998d4594a87f7c5722f222fc45b4fbdcfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/7e6fb8b5422e09cb0be7a707f2f76ee6287efbe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/a15e47ed0ad68cb8274944e5caaf05fd7a530f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/b558d2c7954a4445f0edca8f719e166ebe7a6cfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/ebb90939a3501fb1bdbdf41d1434bf0a63a388bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/1003773a891610c9c6b849621f04ddd96bae0fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/6628909e1e4636b45797f88968682e6efde9c834.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/9529de1481445a2bc2d2748a23070a790ae982c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/c8c5cf95db29a00d92d4a5eeca376433269bbbbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/cde99cdb00eca9432abdf5f91cc07c0448638e55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/d519bf725fd8436d3b91a87a4dded28d0c7530ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/dd2135c3bc2c659dacd9bc8b3180fa99a3661f17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/f690e4118399e1974bf964e761a1dd157bcd912c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/ff65c137ecbd6dc19094784bce0064700f056d0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/36e671019f9221ebb296cf023a12db75a62e8d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/5027b3bb2d3e2be935a367556429dca59f751db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/56aa15f45ccfb8312341621b93fc86fc6a08faa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/6adbdeedb1018f63c92bfcb26ed721ded3664c28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/71595b63b90086ba9e5f29086f55bea8300a2e77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/88c9d1d27cd236c26b68b81ce7950fd3e349981c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/8a8c1f08fdf76de76ff771f458b466463fb3eb8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/a57ff94ff41c13736f84e76cb2cba957e62ead9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/b5b57df9143680f215c48f84db8b24bf46a3343d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/613fe79e612b9ef2a69d95d6160a972ef7a60556.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/7ea637befbb0ef0398dcfe5649777dd41e39d0c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/8c356125b8db641aadac113f6038ec67116f5aed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/983f90150daea258b7d6cbbca7ad4fbbcdaa33a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/9860710c32fd6628715041f6eb364499463d9254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/9a7bc55103e838677f2e7d2173fb52d3f8f412b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/b963231a1dd573c8661b9695717bce574205c92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/cacbf9ed57913d1ab0575883f9d228ad940c5073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/f221635032b71d3e48f1824e30cd2fb641b7cf97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/038187424a0eaf57ba3e6f432d5a8c90bc0ff28b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/13ae730cf3f9e90f6b2f51674dd0753f4c16444a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/1d475e57f48d0fab8bd827bca93fb3d55bd5d654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/85e38c00c6264e765d158b6eb961d46a8d5f712f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/900f88130b7793e902071f6e52a9b85d2908530e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/c748a3fed5d0d5b709166e98cd48f0ef1247f446.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/eac5f148aba5286021b603e5b434b438ac86ba02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/efcc54536aecf62ee04b03838bc7366c94f91579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/f2e1c5c4cc270836292b1be8dfa2405620a53f34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/0437f175e342ea79d784fa4511b00fc88b45fe16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/14b1334f97d9e28c58f0b5fcfc902bb9c46f09e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/31caebd6b810d43b6d4018f0a3021c0005c82788.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/426070e167aac698a1005e1956ba3520e26f7d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/8569360e32504325920b2759265c1e24853ab30f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/a39f2df4dfffa5ff48766c8cdf29e766f015e685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/a7a7a3c6b2b5b7c34313f0dbcc2183b08c5ee5f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/d7d67a64c9fed7fb67dbcfbef79f020950915d96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/e6eb717ccd4d317520e45fa0936ea98d603ca73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/15ee7af204fbc17185469873a465d787fe516baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/530afec9a3c6c6344168a8e1e9f28b95faef3f64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/583ce8af4544d91ec40be4ed5913530e58c65e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/8b52c4111475849cf4578276589eb92d9450f34e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/99c7bb4804592a52b21f5c0102d36aad42bffc2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/a1c063ae0bc6eb1af9f1eb640381381511420d1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/a5929cce0ef4aa35cb21622420cb2c5513597e0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/aee86a9f7c6fc7b89c6aa8827ec5d633ad721aac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/bdb906ae08d1e9ee022303da1fe2ef44f91a57c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/110bc701b380b1fa9a6f6903a01e78805fe7c8ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/1de2854ee9b91a969b9a707cc8485ad01bea3ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/3baac23e8cec17a35bed4f95cadfc475adc50018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/47e4a6c1dc3cb96a5c2cd4a72b707c65f07efa43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/6516ad1d9e170a116777e8eaaccdea5b1717e8d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/929acfacc360de2c494ae3e1380542f902913ce9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/b8675713b33dc4fb897e517cfed1d2e1f07eb29d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/c2970f13f7e46c337e8da428442cd70e3f211765.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/c64738704900564b5b4e853a9350dac25731edc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/1ac2ca6e37d94626e38de61abaab910af2947459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/20410875bd6d909939a41386db150cc65bdf939a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/25a0a0d4664f037b1dd2ac97a236ecc29736e275.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/5b8cf5c52d52fa8e97153262fc896174e9f946fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/5c27769636bc4bf6f2b1abcb502666fd24f7aaad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/6cfdaba69d1544080de67820f5cefb342032340c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/716d92222a959af959a9cce75778a3f8cc183efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/89b4635a56753fe7187c5e62541444fe6227133c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/94eb9103c07fb5edcdb165bd50f82cbfaccd97fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/0e2ca4a01aacfbc482f7ab906df66bd01acfb9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/1f0c06d5b960860c90ef8397932e8bbf01c402ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/5d0b835a9202877d0bffffa18869e44c5a0805ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/8866c1093e1c3ff9916e4b18bdb30f56c2ea7eb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/9cfc4b8bc5b3076360b34c6e4d6a9ce8ed3920f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/bf9fba4336ab355cf36511def1447af0d572581e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/da5d08807f09b4326a4ec875f2f6e58a43d6fb66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/f8a05aeff962f5ddb03398bc3a7e9c0d15ea56cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/ffb044f17a18b2325191890d3ac5e4b53e07b8f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/08c5eae576197eda7507f24f0701ad544ee1a468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/0d8b30e168063b3a1c67a33dd5ffb5e92cc1af17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/3c0d8ac688764da3b3ba466e17965b7253bbc290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/7f8cade2bb97794795fe7a40e69643e94b23f47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/9972be7a1e2b9157d575efd241f6a8a1f2b1a9a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/9de9fd9fc1ae0816ba0c9ce393bd2d16ba0b38fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/cf979900e7e63826a31cbdf39a862461e57faf5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/d572be0fe2c6cbf35f33ecdb6e4182369ba105de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/edcee41775c6b439f6e1e874ae1e34f7f8f9c9ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/1fd9d1689cc8c6822e3493c667f8f5105f3ccfc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/35a93d4e7bc47242c206a950b9431f901f094c1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/5830802a4d741def1503103392854162d1c9dcb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/5b3981c20308e0fab177ff6c33e31f4cd1412ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/91dcc5773597960d67a5c43f0c6e48a965c19481.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/96b3461f210ec38e3226cbdefccf346dd1f8d173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/a71e1a42a45e2ed59997d2b4d1344106eaf5c8a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/be703f39e7ab3e5fda68dd037c2fce22e6807141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/dd09d98d6717c8c4497ffa40010ab691173a5106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/3619986d654374482044eef715f817fd5c06f24a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/67ee8faa1865b51684217b2224f901be7d0c0e46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/7f8b3039eeb1855ec68cf5de731a06a37c0514f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/85ba3348e6ff5a6150bf4867f0e2eda07d3f36dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/8ebc383ab0b4951b2141dfb114dfa78606411ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/ad85ed8eede91e65817399d52d98d550a2da16fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/ccf9253d333de91b07d283d2b48a3dd23e8480ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/de02a768adb374da7ff411f5b2755c13352becaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/f2b5f1bd230364410fcfe857499fb63d8464cad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/16102bffd1106ca5e469ec4bd1eb85ef52ca16a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/569911f41b8885ac6b089b52fa0962d83df6e8c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/94de24d61d664ecbe17e3f1a293b893138c4dd7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/b78467f6d939746e25d346732e699fe424b5d267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/cd2d9dce9a5c5728b9a57771390581bf395cec9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/ced5d7713fa5f9c97181bd7b856339c49ee2451e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/dd532d64365885552cefb85f0c10e399dcadd472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/f1461ed1acdea322fe47730d3cc63f1b87363bb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/f8a7630fbd8cf15ba48ce1f4728a4fa69021098f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/1977842bb3c034c16acfc679f8b3a1b8a2fbf924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/218a3e6f753d642e2524cd2491d1ee846074efd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/3cb599d9cc53bd9cdbaf30377629ce1a49233e60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/4a4616eb85b9f36b7ac5227080a91a3fd213d514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/9d180636b755c8b7d0a119bff6ed4656fa512bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/9f7835183fb17c63788406de196052d5006a8e78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/a0772e0b2396493ce8a41559b76b6a6e98d6550d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/bfb5a0fffd677dcbce31accc8255b15b799387a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/c033250e24f21de1cfdead8a0d2a398fcac8255d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/0136329a14478531c8c67b078b513d28982076a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/040fd54e898dce9cfe17164ebec33e640a803954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/15e5b18b7cc5336b3f022ddc81a3889512647f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/1c3815eaf7ef20118fdca01bf4b3ad45703fc631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/642aaeb981c032dbad58c70cf262441951e52cce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/89f67e7ab6c32f4a9a93c9a80d4d8fc4a33e4d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/b725e36819c428b8d09cb1221584208765fcceb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/d00fa9fa33363ab5bd421ebca5efc3251fcb6ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/d698631fe8c493a959b899acd267ff154fec2e6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/23acfb022756d703ca65fcf8e88139ec4ab7c17b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/2fb651ddf0f97f8d7a95a315e2ecb42371e7c682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/62d9dcaef139c3703000c76349202af9b11efac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/b9f980574e615c0e322cf5b337f3f07181f96d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/bdd4c7268adc7210a4ee738654927141ae0aab9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/dc64944babb485053b8a5db0463aa95285edf510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/eaca8fb29905d40d2852e5edbb368c5e37e85790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/ebe82e98c667c92811466ffa7c154846c7bab313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/ff8f570061bd21124b13dd621832c4525a42c3ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/01ab523503aae4e4ee888e91a5e369f5f7ced90d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/1d692a2d57586377ea3543ab65139722420689c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/3e2c4ce3254010c10e59f94d2f5978047c18330e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/57a27e46249f1f85f11873b4c60c8f018d722f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/722ab2a0a97b17cd215fac47be3339f1b8b54214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/9a177b0c6800b6a98a50e6cd8366b31be19f910f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/c8f109fbcd1de7d101fff7f2e912b52d65b1d1c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/e2949f81092b3e6f172a7fee45101b59fed9f1f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/e3b7bb2db18cf7ba140646ae4df9b6203138e391.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/14049aeec1bfc7621fe26977653b8b169f5a065c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/6d8027ed61ca1b1b419948c57fca18f4a6d3ef99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/852a5112215a3b6518db674c153feecb0a37ad21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/86627f1f7136ee72cd9c1d5cc659d51282d19e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/aaaaeb58ba017c1e245ee94530c23366e03753ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/b7ac9989e6e08eb8e4690dc65cac64f8bd78a274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/bdf0697252833bfbbbe88f1a8f28ffc07c29e9df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/e04654895bf6eedbba4e731010dfe48bd2e1a783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/fc33f0200305bdcda5bf8d4362707d1df895bb81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/099ba6f5ff434b61d6f63af26b8bf8528a0109cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/617dd5b740f9f793fde6e0b2a109e213eb0f58a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/6c4fb90d79b7729b45d174582e454e0c44aeef86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/94360f57ee5fc19874d380385081ebceb78a4943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/a0bdddd23d518b8659a8dd728ae358fbe0360e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/a462ad768b699330d47a32d11baa1d7304fb9097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/b57d2335ab4d37e7c31c14e6e7457fd65be60721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/ddde3f56e7d951c4f7ae3c7ec5a9f5e569bc03ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/f2302b121b006f542ce6ca36c835f4cf1c26f135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/36f7045f24c43fcd3eeb02e31759b45825d64f85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/378111bd1985cc78cba12fce3ea5fd822d70ac81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/4d1f650585e765d04b404902fdc4551bee610298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/6c8ccbff3e411492914300c77b24b4a51cfba41d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/6e6cc9557d7c09ae0bc6cd24a627725366d50989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/7e59c91ec4377de4ccdeff522c41f3d8a242ee5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/cead11b7b863371abd40aa6a818ea22407fb02b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/dfb9d64b908dd950a619ff87a8bf3bafaf09faee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/fd943ba19186349c398e2c425bc146ad364eebee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/03baa0255ffc9cf696c80e132338c2b80fa762dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/127f4de74dee9f0f0cb537b3f38d1eb3c0a5f9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/15c9385edbe98773e7cc5b1acb31fa74b0edd884.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/2f4dcf19874c939629e000dfb224e0c87f110786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/61e84439cb966660e3ca773fcd3944195f50cf0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/6ae9339f701dc375eb06fd005e7efd2808d6e009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/6bf33aba4c67078fb246d1ad7e6c1e7fa257e8d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/b67b05dad64266b21bb22f3231d6475dbcfa7b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/ebe8cfc5055da65768c91374a5ef08a46bec0d88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/405b4c61db92baca3db34386352aef9680ccce6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/62daccc3e38fcd12aa97f0d5efee5d9d7788a59e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/82511ee5a1994eedecd058d55b40fa3db866cf5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/84729c46db4e163d63b1d5b7bcda41a40e564618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/86d68751b522b4b26fee5de0caabcbb173b0070d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/879fdad1b06a9059bf7cc9e85eaec7308dc10a77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/adacf886c16092a307c63cde849d46d9fa15407d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/c9bf8c7acb13bb2060b0ad164bbbe3d0388e5838.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/f2ec8758960df7b57db42e5bfc162fbbfb771125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/791ee78ec34c2d648a2d08dd13f9718697f23282.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/82ee0d1c6a2a789c7c0d430868db47e105653360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/979e353e01a893921dc85db0a1e84007689ed68c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/9ba70104f4aa4148a30a5eb2c5b4c3062460627f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/a01455bda2d4a8dea3aae0ed81f9eaccba0f23fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/a9613e67a21ce41285900e81622c9cb94ad43a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/aae2772e631475dd302b9012180dfa867351a5ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/d11308f94ba150acc721c4a4192d8382a181e9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/fd3df5ab6af848bf376a6dfa6241ad754ecc635f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/10a0371b2cd86f594660e715b5b1ea0213ad2d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/203c430fae57ef5ce4cc930b535a7d2c9950bfb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/239288a183e6752933b890b3ea21ea513b4bf2fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/50696e58b89d1b8ab322fe797f92f8cd51adadc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/7a013d68e08e591f8b1ffaaa831285a133c608b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/86f7fb3c4bfa795392ce6f461b6094caf1c9fa3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/b07fd868198d5c780944fb9a14c2516190223d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/c2d851a0202093320457ccae6fe5872cf5cc2fa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/c91ca1c1e0293580b7f6e91b277efc051090b8d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/10d745158d31e269ad497d8a66bd5563eb14b9e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/2b2e034e2d88db1dacb007ecda5d67eba1c195a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/4c68f19b64f8202f76df3f447fdc5fec21a9dabc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/7067e1533d9a25fbcfb12e22365c39bf2a48a3a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/77f5e83226f952a582b09589065436ef2daa988c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/85671859f751d21b8e48fae8afae9705ad9481b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/c5536f8f6575ed7d2117609c283dee528246d516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/e50c6bf2ce86422f3ddfae9611c390aff6ae757a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/ea75e7bdec31ad6d4ac93e2cfcc28ef47b2794ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/281ede9859f31168a021cff2faac49e8d8ce290b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/2c2baf9fab757dea79b91141ccd754735b62871f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/2ce506d24386d5778c4fdb84ddeac67940294485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/4af49fc4385b59bd30559018e2c047dd3fd85f0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/67d2ba1ce58cbbf32402213cd80db0b867c1834e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/9f706008c571b9ef279effa8454dfd4ef1c073b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/b2b59ddd05288f17633c5c61521d22b6b5de8d41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/d8fd4c73c960db5a74812dec9fc45c17f66fe041.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/f6d97096d0c9a54e4732c7861743dae0e20ff7a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/1ab066f6622034eb08ee5f188878494c1f2461a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/38c35ecaf4155db8155d573f5dce9b1baa2469a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/501472904b43a25606cd256ae2eb1133c4824102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/66fdbfcb1910baf6270bc53507eb5726cf0a6378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/a605c9b847eaacb4d6cb8a492328f32aa056435a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/ab8547655543a7522cb3c95fa80f2c51572434c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/e62657fd7d44fbb2534d0f1ed9260cf0e16186b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/f06de0228d3dee69b876e1fdf1e470509fa74035.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/f7b1c449f2a8cd5abff8aa7716301f9f27a36053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/03961c2360a360e1ad71e176a1f09ad0421fe2fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/209b2bd62a92e2cc97cdb9c603d061e6490406a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/2939f4f0bc207c58b4b90250ba9904c5d9e6a897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/4e16843d39752e6094ebb1e056a9d4c3092638f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/7d98e27122ee9c200f13852d520c95f9e0288306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/97ae59edd64d4b075d2ff9fe8d87d285c179e0f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/c51ed7331899857d4d9cd7aabb16794ca5acf3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/c9091ca35414b6ada89faa9ab688e7e0fc5ecfc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/d841617d75cf1960d6e7445003ad46e5c13ed878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/16673ac1f9ccfd11d3b4825b489ff90018ff3a09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/2c009c248f2f9533d14f2a908aea929622da1f4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/2f1ca563b11a304959e447aea63cd7dca73cde84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/82538d1d0536174596ea6bf20de74c646c2946f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/9ddd527fece21d5f3848b297dd8b1f61e7810b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/a4ce277a7b4d9bf4674b45a6c9a52caf28c654e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/c4f0d2e0e696e65b3b8622031e1a132b49604fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/ec6318b85f033df5c2c922c7e1c0bd4d1c6d8609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/f17a85cd7454d2aa56a93d64c0662c3751aa99a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/01b571981d1974551794268d3002832a0bd11721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/1b20ceb3dbc721d013f86ce65100f33666cd9deb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/285ab6371e4fee521b9aebdf7f420a3035007167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/2a88edeb144f6f41b7490e14190ffea23267b971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/321f0c4a81c573b66cd913637cf246fdc4ffab86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/709798c102c73fb85412ff6fcb5cf39fa067bdd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/7a5fad39ddc66ecfd8a1d6dc7d9e7574aab6bc0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/80a05b13ee7e1da15f115ad6d9be9bd482bf6692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/ba9e87ff7791bf981bbf2ae093eb268e58cb236f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/2027d883f6f58f22f0a527dbf2ebea4e86dac113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/276aa2181d448147ac1b2d21cf8201bdbacf4812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/27933e1c331de2b9bb8ca3a9958787f600ce07f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/2f0c2331cbd2da0023a8188a0005ed440e07a0e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/3d25400c1db1acb67622f7a479c73fc31e22f2e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/7fcb86be4c2565be87f65e5d62b9f3fd7aa5c17e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/c64d60d42c7b5c0ea643a00f39a5096e20f9aa08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/f3536aaac53d58075e63fd69855be871cfd3a65e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/f4adde80fe23ee8dd541447b6f474c604978163b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/012dd021f6cbd2b55283c692813e833557ad4d25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/13f2800b368ccb6abf06fbb096f2042c1a3842f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/31dfe112302292d6d4bda9907ae17fd58270975d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/329901f77dde630994c7e41a9bd45ca9d8d15fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/51a2010cbe8503d8f048c28a2ec35341268ad2c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/79d539e0f471e2854b5a905ec18076c929cf7055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/7fef9a026ecd4b4c53e5e5ea0a91e84d4b07574f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/8a5211efa47c10c39aca5a26df45b9f1e4e66d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/d8e5d12df59dbc4f0e5e9dccf9cf53e79aa940bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/156d9ffb0007f8be0454cd3e44f94ab487c93f8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/275ebda0215930db0466d9cedbb96c2c935a8781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/294198e139487b798a0c0a207d717d3abe890353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/3e40f312fcf40179f3a7ed32213732c5a9499433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/410e33ee9a371696bd1786db48e4880a72d86207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/8b9fc65f7df4a7ed8c9043feaed966c03c18a4e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/c7e12bed8237674c4b39850ae904159b823a0897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/cf90fab17055a85bf61a17a938b227b415ea9adc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/fa2051e01ca9899035fd8cffefca95f205c2b022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/0a8289f6fff3fbcb4659be1c3f988e3f0c54bb96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/0e8ec3499e7cd000347860eb7866310f67abeb77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/2429e28062385756c24f69e8e40b564bcd1d26ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/2f9c3f9638f3abb5da7b1bd8df4897da46718ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/355bc717c616847e494ca030bbae1fb421c240f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/3762ff1e573d2c2300cbf9a18b6379d2f69d478f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/40d635796e872f697e09470b5326240b55e1b7ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/8b25b641306b57ed612f99411929963b077e95db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/8b354005c35a59bf8e9af09ffcac881c708c15b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/49d50e3c23fd9460a833defe564d66917f262dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/52c807d1cd436134d6eb889085e7a124fd1c11dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/740f9a8c3ba8315626500274c9d75cf626f1ec98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/9129d130d1aaa47963f19b2dd8b7c70cb34a2713.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/99bddb9f924b646573ac35aa66fe5a3358c343b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/bef52a1b637701b87a1a0c24d95fddba8a525a6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/cea404136a47a966563034055c0e4209c42b6636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/d5b5d96ac5109873ba5b89a3ede9ff19f079a439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/d79d653f5e2eeef4b471322d1d7b62740f3e2c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/147695464313109d94edf8625dc9f9aea65c9d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/57b490f1a03157bf1f1f729e6c064c70118f695d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/886f95704cae45f946894539f508c10b2775df4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/9ff09bd01e2f4993e6e70a2012e77e8125427b4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/b64179155e845235f195de5e15e65ef8b606f464.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/d266b37e2e14cfc2ab0b37850b8295e433cec997.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/ebe2224da917de966a8cb7dc4e86c5e642adb0d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/f22fb5659ce4bd607020ee7491d803040f46e490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/f57dbae6584a2c37d8b4570627345fd2ba55f872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/337104b9c82021dd8a3097e5fccfe286e0fb95fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/36c1b62528e764b891dff3051ce4462e03b0724d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/3cc3a8a47667c4084f7d7d5d036e567edc65109e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/4bc59b5a0b264f18faeefc3c59932e6ba591377f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/95fcf92b4a1ed5e895cb89e96fcc88c15f5cc259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/b227b34563d17b8073c33a2fa250fc681eb5e49b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/c259bd9656a87838dc4d617bdb7550c0ca36cdac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/d08f8b4a7425db4ce6fa7665b4adce4df8117150.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/de826083ec2d8da743b8ead4201e0c20ad912474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/0090ba7f30524145cb7a9e9d0b07745ec1a336c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/10992fc8038636939baf7ac3e4d00b3232e61536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/2ab75f8e437dd18902fe2f3b075b2a39b2a759ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/4b70fea68167de131a2a1bce27ae0d535b24406c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/4eb65b9d15132d319dc0eec54c63c88c7cba4c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/5cde2995deaf4ebdf4ab0f3177cab71a3ce602f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/8de391b089eccd25f4a884cb6ea15e39c89b74e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/b6ec05e93e0e389d65f1f70259966e2d844e92d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/fc8e2cc1e267fbef08e3dd1ba4dc27b9d6138794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/0d82d5d2841c3cf75cc8ce1e95e2e3fcff597842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/13fdd8ef13b370d79821ea1110b8734371f61ad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/18d829d23145bb3cf6109464bf372f028703d225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/799ddc31b69a410cde689e09a6d21fc87d9f18c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/8176dfa55a2c53ccee09e99f3dd7802f4661100d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/8bdca58aa641a2cb755a744c0b2bb2f09811d1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/9309c8d6a2d90d9a691035801d73abfc2a0dfca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/a4c895e865c207d4ecadfb2d2af4850569a7b7e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/c1f7ca5447935587b848bc376694b503c3a3db48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/1f0d985387917aeb49a8888e3f9868cb2133a3e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/22a7d28ab65c303059080a2ab592c5795551228a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/50ab8f9a85c1fb009d856acd38e8a5c2c04178f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/6ca57704cd33c52b060254a45db6250cf69e95ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/6e46319f46037a0701ca4de1517e3cc14b118031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/708a9f026019503895096477ecde65039c36184f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/7547b0bddde3b749fe7b14de4650d89abdbbde3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/c128b91e1b138d023a3329dc2d3367fe2e7074c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/f1e30379be3473ea6513063930a86d8ca6d48256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/07513acfca69f00eb64a7535098265c8b21caf40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/23ffe98038c2adc062ffc9f973130b36e68e41b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/374d2cc43c7ac1ed2a7c40d89d089e987f96bc56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/3dd602b154f343aeed42d636b5bd123f0da3e41f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/5c01e00f67ad42c5979adc032055242ee3185155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/5cda7407ea2475fc946527b451bc27df1517c443.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/654e8ce5892b86980abdae1b9e2b3ee947525cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/d2687d8bcc3eaae4953561699a3f98cf20a82710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/fc647730393344cdabc5b347e164ae16907602cd.jsonl.zst + +dedupe: + name: dedupe_url + documents: + attribute_name: dedupe_url + key: $.metadata.url + skip_empty: true + + +bloom_filter: + file: /tmp/dedupe-url/cccc-CC-MAIN-2024-18/filter.bloom + read_only: false + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/dedupe-url/cccc-CC-MAIN-2024-18/input + output: /tmp/dedupe-url/cccc-CC-MAIN-2024-18/output diff --git a/configs/cccc/language_counts.yaml b/configs/cccc/language_counts.yaml new file mode 100644 index 00000000..697aba45 --- /dev/null +++ b/configs/cccc/language_counts.yaml @@ -0,0 +1,71 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-40/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-40/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-24/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-10/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-10/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-51/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-51/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-43/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-43/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-09/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-30/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-30/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-30/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-30/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-26/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-26/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/*.zst + +taggers: + - whitespace_tokenizer_v1 + - ft_lang_id_1e2 + - tokenizer_repetitions_v2r2 + - c4_v2 + - gopher_v1 + +processes: ${d.procs:} diff --git a/configs/cccc/mix-v2/CC-MAIN-2013-20.yaml b/configs/cccc/mix-v2/CC-MAIN-2013-20.yaml new file mode 100644 index 00000000..085716b0 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2013-20.yaml @@ -0,0 +1,61 @@ + +streams: + - name: cccc-CC-MAIN-2013-20 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20/cccc-CC-MAIN-2013-20-0027.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2013-20 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2013-20/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2013-20/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2013-48.yaml b/configs/cccc/mix-v2/CC-MAIN-2013-48.yaml new file mode 100644 index 00000000..9cb8fe79 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2013-48.yaml @@ -0,0 +1,66 @@ + +streams: + - name: cccc-CC-MAIN-2013-48 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48/cccc-CC-MAIN-2013-48-0032.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2013-48 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2013-48/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2013-48/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2014-10.yaml b/configs/cccc/mix-v2/CC-MAIN-2014-10.yaml new file mode 100644 index 00000000..826b4f82 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2014-10.yaml @@ -0,0 +1,70 @@ + +streams: + - name: cccc-CC-MAIN-2014-10 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10/cccc-CC-MAIN-2014-10-0036.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2014-10 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-10/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-10/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2014-15.yaml b/configs/cccc/mix-v2/CC-MAIN-2014-15.yaml new file mode 100644 index 00000000..cc8144b5 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2014-15.yaml @@ -0,0 +1,79 @@ + +streams: + - name: cccc-CC-MAIN-2014-15 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15/cccc-CC-MAIN-2014-15-0045.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2014-15 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-15/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-15/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2014-23.yaml b/configs/cccc/mix-v2/CC-MAIN-2014-23.yaml new file mode 100644 index 00000000..3b433baa --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2014-23.yaml @@ -0,0 +1,87 @@ + +streams: + - name: cccc-CC-MAIN-2014-23 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0052.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23/cccc-CC-MAIN-2014-23-0053.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2014-23 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-23/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-23/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2014-35.yaml b/configs/cccc/mix-v2/CC-MAIN-2014-35.yaml new file mode 100644 index 00000000..36b7d929 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2014-35.yaml @@ -0,0 +1,77 @@ + +streams: + - name: cccc-CC-MAIN-2014-35 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35/cccc-CC-MAIN-2014-35-0043.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2014-35 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-35/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-35/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2014-41.yaml b/configs/cccc/mix-v2/CC-MAIN-2014-41.yaml new file mode 100644 index 00000000..f4d8c6a6 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2014-41.yaml @@ -0,0 +1,75 @@ + +streams: + - name: cccc-CC-MAIN-2014-41 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41/cccc-CC-MAIN-2014-41-0041.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2014-41 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-41/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-41/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2014-42.yaml b/configs/cccc/mix-v2/CC-MAIN-2014-42.yaml new file mode 100644 index 00000000..98b974ef --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2014-42.yaml @@ -0,0 +1,98 @@ + +streams: + - name: cccc-CC-MAIN-2014-42 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0052.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0053.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0054.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0055.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0056.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0057.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0058.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0059.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0060.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0061.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0062.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0063.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42/cccc-CC-MAIN-2014-42-0064.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2014-42 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-42/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-42/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2014-49.yaml b/configs/cccc/mix-v2/CC-MAIN-2014-49.yaml new file mode 100644 index 00000000..9f45322f --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2014-49.yaml @@ -0,0 +1,66 @@ + +streams: + - name: cccc-CC-MAIN-2014-49 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49/cccc-CC-MAIN-2014-49-0032.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2014-49 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-49/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-49/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2014-52.yaml b/configs/cccc/mix-v2/CC-MAIN-2014-52.yaml new file mode 100644 index 00000000..c9790c18 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2014-52.yaml @@ -0,0 +1,67 @@ + +streams: + - name: cccc-CC-MAIN-2014-52 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52/cccc-CC-MAIN-2014-52-0033.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2014-52 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-52/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-52/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2015-06.yaml b/configs/cccc/mix-v2/CC-MAIN-2015-06.yaml new file mode 100644 index 00000000..64250d43 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2015-06.yaml @@ -0,0 +1,61 @@ + +streams: + - name: cccc-CC-MAIN-2015-06 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06/cccc-CC-MAIN-2015-06-0027.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2015-06 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-06/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-06/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2015-11.yaml b/configs/cccc/mix-v2/CC-MAIN-2015-11.yaml new file mode 100644 index 00000000..d6118fc1 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2015-11.yaml @@ -0,0 +1,66 @@ + +streams: + - name: cccc-CC-MAIN-2015-11 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11/cccc-CC-MAIN-2015-11-0032.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2015-11 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-11/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-11/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2015-14.yaml b/configs/cccc/mix-v2/CC-MAIN-2015-14.yaml new file mode 100644 index 00000000..29a822aa --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2015-14.yaml @@ -0,0 +1,63 @@ + +streams: + - name: cccc-CC-MAIN-2015-14 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14/cccc-CC-MAIN-2015-14-0029.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2015-14 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-14/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-14/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2015-18.yaml b/configs/cccc/mix-v2/CC-MAIN-2015-18.yaml new file mode 100644 index 00000000..3d5419e0 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2015-18.yaml @@ -0,0 +1,71 @@ + +streams: + - name: cccc-CC-MAIN-2015-18 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18/cccc-CC-MAIN-2015-18-0037.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2015-18 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-18/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-18/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2015-22.yaml b/configs/cccc/mix-v2/CC-MAIN-2015-22.yaml new file mode 100644 index 00000000..3b5ceaeb --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2015-22.yaml @@ -0,0 +1,70 @@ + +streams: + - name: cccc-CC-MAIN-2015-22 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22/cccc-CC-MAIN-2015-22-0036.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2015-22 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-22/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-22/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2015-27.yaml b/configs/cccc/mix-v2/CC-MAIN-2015-27.yaml new file mode 100644 index 00000000..55aad68e --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2015-27.yaml @@ -0,0 +1,64 @@ + +streams: + - name: cccc-CC-MAIN-2015-27 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27/cccc-CC-MAIN-2015-27-0030.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2015-27 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-27/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-27/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2016-07.yaml b/configs/cccc/mix-v2/CC-MAIN-2016-07.yaml new file mode 100644 index 00000000..956d7d17 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2016-07.yaml @@ -0,0 +1,68 @@ + +streams: + - name: cccc-CC-MAIN-2016-07 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07/cccc-CC-MAIN-2016-07-0034.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2016-07 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2016-07/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2016-07/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2016-18.yaml b/configs/cccc/mix-v2/CC-MAIN-2016-18.yaml new file mode 100644 index 00000000..95a59be2 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2016-18.yaml @@ -0,0 +1,59 @@ + +streams: + - name: cccc-CC-MAIN-2016-18 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18/cccc-CC-MAIN-2016-18-0025.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2016-18 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2016-18/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2016-18/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2016-22.yaml b/configs/cccc/mix-v2/CC-MAIN-2016-22.yaml new file mode 100644 index 00000000..a535e60b --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2016-22.yaml @@ -0,0 +1,58 @@ + +streams: + - name: cccc-CC-MAIN-2016-22 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22/cccc-CC-MAIN-2016-22-0024.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2016-22 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2016-22/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2016-22/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2016-26.yaml b/configs/cccc/mix-v2/CC-MAIN-2016-26.yaml new file mode 100644 index 00000000..915696dd --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2016-26.yaml @@ -0,0 +1,34 @@ + +streams: + - name: cccc-CC-MAIN-2016-26 + documents: + + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2016-26 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2016-26/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2016-26/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2016-30.yaml b/configs/cccc/mix-v2/CC-MAIN-2016-30.yaml new file mode 100644 index 00000000..b9a7b5ac --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2016-30.yaml @@ -0,0 +1,34 @@ + +streams: + - name: cccc-CC-MAIN-2016-30 + documents: + + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2016-30 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2016-30/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2016-30/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2017-04.yaml b/configs/cccc/mix-v2/CC-MAIN-2017-04.yaml new file mode 100644 index 00000000..8a343009 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2017-04.yaml @@ -0,0 +1,70 @@ + +streams: + - name: cccc-CC-MAIN-2017-04 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04/cccc-CC-MAIN-2017-04-0036.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2017-04 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-04/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-04/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2017-09.yaml b/configs/cccc/mix-v2/CC-MAIN-2017-09.yaml new file mode 100644 index 00000000..a026e5ca --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2017-09.yaml @@ -0,0 +1,77 @@ + +streams: + - name: cccc-CC-MAIN-2017-09 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09/cccc-CC-MAIN-2017-09-0043.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2017-09 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-09/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-09/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2017-13.yaml b/configs/cccc/mix-v2/CC-MAIN-2017-13.yaml new file mode 100644 index 00000000..7ba14d28 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2017-13.yaml @@ -0,0 +1,82 @@ + +streams: + - name: cccc-CC-MAIN-2017-13 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13/cccc-CC-MAIN-2017-13-0048.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2017-13 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-13/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-13/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2017-17.yaml b/configs/cccc/mix-v2/CC-MAIN-2017-17.yaml new file mode 100644 index 00000000..664b2a6a --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2017-17.yaml @@ -0,0 +1,84 @@ + +streams: + - name: cccc-CC-MAIN-2017-17 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17/cccc-CC-MAIN-2017-17-0050.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2017-17 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-17/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-17/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2017-22.yaml b/configs/cccc/mix-v2/CC-MAIN-2017-22.yaml new file mode 100644 index 00000000..b829d235 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2017-22.yaml @@ -0,0 +1,82 @@ + +streams: + - name: cccc-CC-MAIN-2017-22 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22/cccc-CC-MAIN-2017-22-0048.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2017-22 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-22/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-22/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2017-26.yaml b/configs/cccc/mix-v2/CC-MAIN-2017-26.yaml new file mode 100644 index 00000000..b57e2bbe --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2017-26.yaml @@ -0,0 +1,87 @@ + +streams: + - name: cccc-CC-MAIN-2017-26 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0052.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26/cccc-CC-MAIN-2017-26-0053.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2017-26 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-26/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-26/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2017-30.yaml b/configs/cccc/mix-v2/CC-MAIN-2017-30.yaml new file mode 100644 index 00000000..c9acd27a --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2017-30.yaml @@ -0,0 +1,34 @@ + +streams: + - name: cccc-CC-MAIN-2017-30 + documents: + + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2017-30 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-30/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-30/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2017-51.yaml b/configs/cccc/mix-v2/CC-MAIN-2017-51.yaml new file mode 100644 index 00000000..c6e2e9ac --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2017-51.yaml @@ -0,0 +1,54 @@ + +streams: + - name: cccc-CC-MAIN-2017-51 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51/cccc-CC-MAIN-2017-51-0020.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2017-51 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-51/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-51/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2018-09.yaml b/configs/cccc/mix-v2/CC-MAIN-2018-09.yaml new file mode 100644 index 00000000..052cc6df --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2018-09.yaml @@ -0,0 +1,34 @@ + +streams: + - name: cccc-CC-MAIN-2018-09 + documents: + + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2018-09 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-09/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-09/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2018-13.yaml b/configs/cccc/mix-v2/CC-MAIN-2018-13.yaml new file mode 100644 index 00000000..2972fedc --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2018-13.yaml @@ -0,0 +1,76 @@ + +streams: + - name: cccc-CC-MAIN-2018-13 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13/cccc-CC-MAIN-2018-13-0042.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2018-13 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-13/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-13/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2018-22.yaml b/configs/cccc/mix-v2/CC-MAIN-2018-22.yaml new file mode 100644 index 00000000..8e472f0f --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2018-22.yaml @@ -0,0 +1,69 @@ + +streams: + - name: cccc-CC-MAIN-2018-22 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22/cccc-CC-MAIN-2018-22-0035.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2018-22 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-22/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-22/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2018-26.yaml b/configs/cccc/mix-v2/CC-MAIN-2018-26.yaml new file mode 100644 index 00000000..5b2a2eee --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2018-26.yaml @@ -0,0 +1,74 @@ + +streams: + - name: cccc-CC-MAIN-2018-26 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26/cccc-CC-MAIN-2018-26-0040.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2018-26 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-26/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-26/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2018-30.yaml b/configs/cccc/mix-v2/CC-MAIN-2018-30.yaml new file mode 100644 index 00000000..f6b1241a --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2018-30.yaml @@ -0,0 +1,78 @@ + +streams: + - name: cccc-CC-MAIN-2018-30 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30/cccc-CC-MAIN-2018-30-0044.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2018-30 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-30/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-30/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2018-34.yaml b/configs/cccc/mix-v2/CC-MAIN-2018-34.yaml new file mode 100644 index 00000000..04facae9 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2018-34.yaml @@ -0,0 +1,64 @@ + +streams: + - name: cccc-CC-MAIN-2018-34 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34/cccc-CC-MAIN-2018-34-0030.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2018-34 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-34/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-34/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2018-47.yaml b/configs/cccc/mix-v2/CC-MAIN-2018-47.yaml new file mode 100644 index 00000000..3d0d4323 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2018-47.yaml @@ -0,0 +1,64 @@ + +streams: + - name: cccc-CC-MAIN-2018-47 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47/cccc-CC-MAIN-2018-47-0030.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2018-47 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-47/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-47/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2018-51.yaml b/configs/cccc/mix-v2/CC-MAIN-2018-51.yaml new file mode 100644 index 00000000..3d9f1b93 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2018-51.yaml @@ -0,0 +1,69 @@ + +streams: + - name: cccc-CC-MAIN-2018-51 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51/cccc-CC-MAIN-2018-51-0035.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2018-51 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-51/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-51/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2019-04.yaml b/configs/cccc/mix-v2/CC-MAIN-2019-04.yaml new file mode 100644 index 00000000..53528b88 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2019-04.yaml @@ -0,0 +1,69 @@ + +streams: + - name: cccc-CC-MAIN-2019-04 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04/cccc-CC-MAIN-2019-04-0035.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2019-04 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-04/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-04/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2019-09.yaml b/configs/cccc/mix-v2/CC-MAIN-2019-09.yaml new file mode 100644 index 00000000..4bc70413 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2019-09.yaml @@ -0,0 +1,68 @@ + +streams: + - name: cccc-CC-MAIN-2019-09 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09/cccc-CC-MAIN-2019-09-0034.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2019-09 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-09/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-09/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2019-13.yaml b/configs/cccc/mix-v2/CC-MAIN-2019-13.yaml new file mode 100644 index 00000000..ecf92074 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2019-13.yaml @@ -0,0 +1,64 @@ + +streams: + - name: cccc-CC-MAIN-2019-13 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13/cccc-CC-MAIN-2019-13-0030.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2019-13 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-13/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-13/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2019-30.yaml b/configs/cccc/mix-v2/CC-MAIN-2019-30.yaml new file mode 100644 index 00000000..af3d0f47 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2019-30.yaml @@ -0,0 +1,74 @@ + +streams: + - name: cccc-CC-MAIN-2019-30 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30/cccc-CC-MAIN-2019-30-0040.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2019-30 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-30/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-30/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2019-35.yaml b/configs/cccc/mix-v2/CC-MAIN-2019-35.yaml new file mode 100644 index 00000000..868214b6 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2019-35.yaml @@ -0,0 +1,84 @@ + +streams: + - name: cccc-CC-MAIN-2019-35 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35/cccc-CC-MAIN-2019-35-0050.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2019-35 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-35/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-35/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2019-39.yaml b/configs/cccc/mix-v2/CC-MAIN-2019-39.yaml new file mode 100644 index 00000000..37710456 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2019-39.yaml @@ -0,0 +1,80 @@ + +streams: + - name: cccc-CC-MAIN-2019-39 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39/cccc-CC-MAIN-2019-39-0046.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2019-39 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-39/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-39/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2019-43.yaml b/configs/cccc/mix-v2/CC-MAIN-2019-43.yaml new file mode 100644 index 00000000..3c12614d --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2019-43.yaml @@ -0,0 +1,34 @@ + +streams: + - name: cccc-CC-MAIN-2019-43 + documents: + + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2019-43 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-43/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-43/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2019-51.yaml b/configs/cccc/mix-v2/CC-MAIN-2019-51.yaml new file mode 100644 index 00000000..4abb6548 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2019-51.yaml @@ -0,0 +1,34 @@ + +streams: + - name: cccc-CC-MAIN-2019-51 + documents: + + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2019-51 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-51/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-51/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2020-10.yaml b/configs/cccc/mix-v2/CC-MAIN-2020-10.yaml new file mode 100644 index 00000000..d2a3428d --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2020-10.yaml @@ -0,0 +1,34 @@ + +streams: + - name: cccc-CC-MAIN-2020-10 + documents: + + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2020-10 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2020-10/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2020-10/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2020-24.yaml b/configs/cccc/mix-v2/CC-MAIN-2020-24.yaml new file mode 100644 index 00000000..822852d4 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2020-24.yaml @@ -0,0 +1,34 @@ + +streams: + - name: cccc-CC-MAIN-2020-24 + documents: + + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2020-24 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2020-24/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2020-24/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2020-29.yaml b/configs/cccc/mix-v2/CC-MAIN-2020-29.yaml new file mode 100644 index 00000000..0266fa2c --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2020-29.yaml @@ -0,0 +1,86 @@ + +streams: + - name: cccc-CC-MAIN-2020-29 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0048.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0049.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0050.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0051.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29/cccc-CC-MAIN-2020-29-0052.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2020-29 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2020-29/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2020-29/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2020-34.yaml b/configs/cccc/mix-v2/CC-MAIN-2020-34.yaml new file mode 100644 index 00000000..ffe5c26f --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2020-34.yaml @@ -0,0 +1,80 @@ + +streams: + - name: cccc-CC-MAIN-2020-34 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34/cccc-CC-MAIN-2020-34-0046.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2020-34 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2020-34/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2020-34/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2020-40.yaml b/configs/cccc/mix-v2/CC-MAIN-2020-40.yaml new file mode 100644 index 00000000..68c278aa --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2020-40.yaml @@ -0,0 +1,34 @@ + +streams: + - name: cccc-CC-MAIN-2020-40 + documents: + + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2020-40 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2020-40/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2020-40/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2021-17.yaml b/configs/cccc/mix-v2/CC-MAIN-2021-17.yaml new file mode 100644 index 00000000..99c5115f --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2021-17.yaml @@ -0,0 +1,50 @@ + +streams: + - name: cccc-CC-MAIN-2021-17 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17/cccc-CC-MAIN-2021-17-0016.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2021-17 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2021-17/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2021-17/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2021-39.yaml b/configs/cccc/mix-v2/CC-MAIN-2021-39.yaml new file mode 100644 index 00000000..6dc2f33c --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2021-39.yaml @@ -0,0 +1,75 @@ + +streams: + - name: cccc-CC-MAIN-2021-39 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39/cccc-CC-MAIN-2021-39-0041.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2021-39 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2021-39/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2021-39/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2021-43.yaml b/configs/cccc/mix-v2/CC-MAIN-2021-43.yaml new file mode 100644 index 00000000..91267626 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2021-43.yaml @@ -0,0 +1,82 @@ + +streams: + - name: cccc-CC-MAIN-2021-43 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0045.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0046.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0047.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43/cccc-CC-MAIN-2021-43-0048.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2021-43 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2021-43/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2021-43/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2021-49.yaml b/configs/cccc/mix-v2/CC-MAIN-2021-49.yaml new file mode 100644 index 00000000..327d248f --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2021-49.yaml @@ -0,0 +1,70 @@ + +streams: + - name: cccc-CC-MAIN-2021-49 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49/cccc-CC-MAIN-2021-49-0036.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2021-49 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2021-49/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2021-49/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2022-05.yaml b/configs/cccc/mix-v2/CC-MAIN-2022-05.yaml new file mode 100644 index 00000000..c83d9158 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2022-05.yaml @@ -0,0 +1,75 @@ + +streams: + - name: cccc-CC-MAIN-2022-05 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/cccc-CC-MAIN-2022-05-0041.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2022-05 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2022-05/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2022-05/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2023-06.yaml b/configs/cccc/mix-v2/CC-MAIN-2023-06.yaml new file mode 100644 index 00000000..c717b72a --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2023-06.yaml @@ -0,0 +1,79 @@ + +streams: + - name: cccc-CC-MAIN-2023-06 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0043.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0044.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06/cccc-CC-MAIN-2023-06-0045.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2023-06 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2023-06/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2023-06/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2023-14.yaml b/configs/cccc/mix-v2/CC-MAIN-2023-14.yaml new file mode 100644 index 00000000..1b0bed53 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2023-14.yaml @@ -0,0 +1,76 @@ + +streams: + - name: cccc-CC-MAIN-2023-14 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14/cccc-CC-MAIN-2023-14-0042.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2023-14 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2023-14/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2023-14/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2023-23.yaml b/configs/cccc/mix-v2/CC-MAIN-2023-23.yaml new file mode 100644 index 00000000..28cba9cf --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2023-23.yaml @@ -0,0 +1,76 @@ + +streams: + - name: cccc-CC-MAIN-2023-23 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23/cccc-CC-MAIN-2023-23-0042.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2023-23 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2023-23/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2023-23/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2023-40.yaml b/configs/cccc/mix-v2/CC-MAIN-2023-40.yaml new file mode 100644 index 00000000..7a9bffa8 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2023-40.yaml @@ -0,0 +1,34 @@ + +streams: + - name: cccc-CC-MAIN-2023-40 + documents: + + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2023-40 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2023-40/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2023-40/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2023-50.yaml b/configs/cccc/mix-v2/CC-MAIN-2023-50.yaml new file mode 100644 index 00000000..3099e758 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2023-50.yaml @@ -0,0 +1,77 @@ + +streams: + - name: cccc-CC-MAIN-2023-50 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0038.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0039.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0040.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0041.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0042.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50/cccc-CC-MAIN-2023-50-0043.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2023-50 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2023-50/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2023-50/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2024-10.yaml b/configs/cccc/mix-v2/CC-MAIN-2024-10.yaml new file mode 100644 index 00000000..232d288d --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2024-10.yaml @@ -0,0 +1,72 @@ + +streams: + - name: cccc-CC-MAIN-2024-10 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0035.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0036.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0037.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10/cccc-CC-MAIN-2024-10-0038.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2024-10 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2024-10/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2024-10/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mix-v2/CC-MAIN-2024-18.yaml b/configs/cccc/mix-v2/CC-MAIN-2024-18.yaml new file mode 100644 index 00000000..e529d0c9 --- /dev/null +++ b/configs/cccc/mix-v2/CC-MAIN-2024-18.yaml @@ -0,0 +1,69 @@ + +streams: + - name: cccc-CC-MAIN-2024-18 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0000.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0001.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0002.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0003.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0004.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0005.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0006.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0007.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0008.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0009.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0010.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0011.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0012.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0014.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0015.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0016.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0017.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0018.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0019.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0020.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0021.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0022.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0023.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0024.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0025.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0026.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0027.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0029.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0030.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0031.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0032.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0033.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0034.json.gz + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18/cccc-CC-MAIN-2024-18-0035.json.gz + attributes: + - dedupe_doc + - dedupe_para + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v2_nc_nd_fix/documents/CC-MAIN-2024-18 + discard_fields: + - attributes + filter: + syntax: jq + exclude: + # Exact duplicates + - .attributes.dedupe_doc != null + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_|_nd_"))) | length > 0) + + # Fuzzy duplicates with above 0.8 of shared ngrams + - >- + (.attributes.dedupe_para | length > 0) and + ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) >= 0.8) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2024-18/mix-v2/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2024-18/mix-v2/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2013-20.yaml b/configs/cccc/mixer/CC-MAIN-2013-20.yaml new file mode 100644 index 00000000..f25ae53d --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2013-20.yaml @@ -0,0 +1,425 @@ + +streams: + - name: cccc-CC-MAIN-2013-20 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696381249/warc/46805c0d8c9fc6af75faa2d92ba0c9de31ea0e0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696381630/warc/bda4ecb19f1d2e1e1139b02c3201a3cda6bcd5a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382185/warc/5bb9daa063a27ec2c05044683bf9a0d32aecccd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382261/warc/0b53c043d81a6c4d331c5de460aad479b34d5c7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382360/warc/20e49e1879eae233f8d42f77b9046b9e28a8f6a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382396/warc/7ccf2df755e7629bb81cae028ed0cec9d67916d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382398/warc/32791d0fab3bba1846160fc7924d7f04596d67cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382450/warc/12fa4ae43179e174395fd2e8644772882cdf62bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382503/warc/697f80c09ceeaac80ccb2928af5f6cec416a6158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382560/warc/14862c09ae0d6bcab6c9bd16eaed35723550f57a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382584/warc/3dd5daf20b67d0381572739190b5b3aae081345b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382705/warc/e59c5cdb46820da52e8487275b88a4dd71a99cb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382764/warc/76301d96618f61a46453753d4b5e48aa5dc96b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382851/warc/837eef8e0a409e2572139030f1a65ecc301324ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382892/warc/bfe49064523bd1a7b91a5d42d4563dfcd9ebf600.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382917/warc/f6097c04db2f886348eae3a901adb3a661fa1be4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382920/warc/2fdb48eb9fadf62ac4a0f1611d8d25171915e79a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696382989/warc/829aa936d114953ddc9e526e10ce1464be205d96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383077/warc/588407816cd8c6ef112ca11002fa30b043b69357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383081/warc/09eb6a85cab1f1e31b15361b8208173883b4e778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383156/warc/73ccb45aef841fdbbfd4e3424b5173243b05854b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383160/warc/6a3a4f46ff7173a029ea2b1f87006732d659d791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383218/warc/6e8b6879ad43d66f27cb7905b551605ba9cb6f4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383259/warc/0d60ee4cf5cde9f497a5e3190d87429cd0244433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383263/warc/dfcfeedb25f450e759f50ec0da0fab5110f9166c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696383508/warc/dfa8ea0fb40ff878b86e8be5f786ddc556bbfeed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696384181/warc/ae982f3423f26d51f36f6d87fb38766c2507b166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696384213/warc/f85a944fa0b52df084ac1f46c6194eac8bcb7784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368696400149/warc/9c25506ba83e6fd7033b78b5e5bf9ba9897bc649.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697232084/warc/49980cd4360675ee58bac843dc98e177d49402eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697380733/warc/0da287c90e344c81bdb2a65c5f608b1c108aa87b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697420704/warc/52cf74cb438faa1c9e9aa9a8b49466c94ee70a86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697442043/warc/998a5ea6b2b3119b3efe122ef733619ccc83af65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697503739/warc/dc01d6563980ba29c36d399cae7b25d8c186e6fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697552127/warc/127c755c6b65e97a01f025d4cdddca2a901dbbb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697681504/warc/9db9b018c69e0ca3db1b22c95da9e14c998f4f4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697745221/warc/011d1a8b3af716cc99d2f309e15255bee8491e79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697772439/warc/3b660b9f97b36826f2c3145be88e9ca5418ad465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697843948/warc/a960f65435f1fc8807fa0324ebe0090c34f553b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697917013/warc/b8759ec2a80796c4893d24c16c7d9159a3b94c85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368697974692/warc/6170fcf8c5ace5a92dc0919819e599af896822d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698017611/warc/9fbe35a03cf6a0bc5d08dd50ad2e4590644ff70b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698063918/warc/c00f51d0c5ab6fefbf197487a45f8bc61b578b2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698080772/warc/e4ff76de84939b939de894cd72394979d5aac68e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698090094/warc/85d02ebbfcfe43114066ce65ed516c2a3cdf9987.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698104521/warc/6c09fb5234ff26d717264c9c3f370bd3eca2efaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698141028/warc/f9283cf0e4948fab179d5ad422f03ad2f4bebbe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698150793/warc/ce7d36518318957a97d7a87515a7aca33e41d98f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698196686/warc/0c719e955be656f96ccd7c4f1f12318a2ee0df99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698203920/warc/d2b987ab19d98fa9a8bf9c3523ba36f2b113fda1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698207393/warc/2fa0f8ba4d97387b60bc71ba2c0bbbef823fa64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698222543/warc/77573a4051386b9b54c08d9629dab8e062774ac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698238192/warc/0a4e424c0234c400371ec13f53690814e11ace73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698289937/warc/5f66cc3816b372940f007dc511322b077919765e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698354227/warc/10414f9adee91643c16a4baa9bac6a981d6253b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698411148/warc/19f88d5dafc307b9777bc38084447282e55b8c48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698493317/warc/4650ac43ab46d6bd8191ee330378f3c318386187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698554957/warc/81e4976cb11adec8761e86de329d4ef1cbc222fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698646863/warc/9711e11d786077aa6bf018fb7579e12a729f2fb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698693943/warc/a28a747edf98ec54d3e80374abb48c393118c043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698924319/warc/54d5c1aba654e117f53687198ad40fca5945e6b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368698958430/warc/52349e4fee66e33dcffde3067214e9997e2c22c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699036375/warc/34d360c51827de7a2e2b4b946641ca7f6d1f64d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699056351/warc/499a7620b383522235eaa1942fe8f2275b469b94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699068791/warc/cd80ad7a337c76125c7d1a6099ea00fcf348f0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699113041/warc/9ce0842299167f8f4f84424bf1d1773638852b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699138006/warc/80a17bd9cc8c1571906613af5f0f79a325d9fdee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699186520/warc/d58bd87385e120661ce192d7c0eb1a13a5d59273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699201808/warc/407599ecefb153a0968c00bf7b6e3d4f02d7cad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699238089/warc/ab461613e84815296e5535972a944fd93792d1ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699273641/warc/038e4ef959d153cbd51536a9b850d5d174969f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699632815/warc/c09c6c77d6b751322ba498833e668c3a73d0167f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699675907/warc/ede2a52f7209e75793b2a832d715de5764a79b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699684236/warc/e6d3f6e25f69ed26305be3370bd06aecebfeaf20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699730479/warc/72a425b8e2112d655a9dcb5d9006c245eb7463df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699755211/warc/dd1b98e1da1f845f6ffeb655727cea0b7f91331f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699776315/warc/f579ed5039d0fb256166ec85861c80b5d38cf874.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699798457/warc/7cd539e57d1eae519167b40a084bb37d96ed3bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699812416/warc/e9be901b0cb1737385f26b496f22c413bd82e7a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699856050/warc/9774f4091932ff78018c0499cd497d0dfd6e36df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699881956/warc/114f1810414924e3560bbb2b4d9f031253b2815c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699899882/warc/207ea2264258d050e4aca234c71e46e284937520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699924051/warc/7515baf8421ea8bb9329fa00af8bef559d790716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368699977678/warc/6a8c9544ed9c046f2e77701d5b32b0344ec73b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700014987/warc/c348fecf5796e0a4983ea98c026d5c7a148c70e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700074077/warc/a9a78f31456541d7393ac8916ad0f5094302dc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700107557/warc/858fe1b8eec0990924f04735cf194810583a9f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700132256/warc/85115016d2a8c707926103df0085008effac0c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700168711/warc/0ac9e2155984d9f807bfcdef60c792188aa30b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700212265/warc/6a0bfc3c4fd59c307caf1ba0567baf7f4d0ce533.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700264179/warc/2ea94751817d62a54570f94215c6cf270a14ddd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700380063/warc/7b99931a4d55d3288cae8a910ae4ea2739614630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700438490/warc/fb58aa83fb652296341e8376fae1e3ea188e9ce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700477029/warc/2ffbc54942ce400169d0c34f7e7d8e782f0d5b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700497024/warc/7797ea9bd1f26e1b2755b8c73c9c14d834f39586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700563008/warc/4d8493e05f786baa437870d2ce8b4219b0ff042d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700626424/warc/b3221b1810053c98e8d2c1f911da5fa89f0a903f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700795821/warc/7725b160bbd9a96142e62daba7e44760b41b3256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700842908/warc/ce8c85c4f4b6bea199807f9bf2fb4d461af786b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700871976/warc/05d7da0e4c0d4ec1471ea60621aec91fb0bf8d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700958435/warc/83c9e1c7917a62c9981476db8a638bd69e7ec6a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368700984410/warc/01d522f74fe6a9d5db4b404766ba2d71c640c7dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701063060/warc/66ae4555553458b5ab33a141d1107cb29ff30918.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701153213/warc/3ae64dc6db96db0e8329791d9b0e257b6e53e0bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701233842/warc/b76b23a02692752356b51701afdeda040cc8b197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701281163/warc/4db941df9d6246a47a2089d7acd3435d57f92962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701314683/warc/5c2c2eed56416afab684fe6e41874d77d13743fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701370254/warc/a9053040cb131f5a7112080893c9340734ea8d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701409268/warc/d354f4830d85d027d9dea12ea9c3acff7ff6bb01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701445114/warc/3e73ff0f4bc3d9fe80669b27f7944cabd411c7f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701459211/warc/a23df5008361aeedfe203e1e2dd1e837e6e10aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701508530/warc/648413bb95c56e1422cce5a4a26cf89a68111cce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701543416/warc/ed7e516b96ab22ab605aba247f49d2b75a5c591a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701562534/warc/8581e6328931c7cc07cb1f7308e59fc9a76d18cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701577515/warc/6019f83109d7ed3cf0411e8eeeb872865ece52f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701614932/warc/009f11b70f47eaa6f8a27210f64690c5dda5ab21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701638778/warc/d979eafb0afec22f3248ed1eeef9e56c24b5868f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701670866/warc/4992ab3c9510ce53bd11ebed664d37c1241f5477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701760529/warc/6aba771e0f3f68ecf2aa336d0b2a38ab2feccc6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701806508/warc/4481d3bd53b269684539c162a07dce6880717e1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701852492/warc/7cfedf7527fda75993ecd569e875ed947799ba6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701910820/warc/e88eacfac85db5ab301e487a513c29896ed8f8c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368701943764/warc/0cd3833dbd51439a06757644e52a5b26b2b7e0af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702019913/warc/d162c636969d362d67341723d9555409446c31d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702127714/warc/cd118e3bfd443c7de791bf143df7591e868c4084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702185502/warc/59f9df69b9e771d5542001bac54b581a9c555d80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702298845/warc/fee3da7e30f6c00c1c43496ef08c8e7a46e82d4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702414478/warc/f96e8cfe614aac2c197045c62fbb4061e06fbebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702444272/warc/6811866ffc78ae8c348fcf939f5032534bacffb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702447607/warc/9f61ddeb3f80ecdff5058fa3e57d0162a7833166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702448584/warc/a77bd48f436982e2c2e6dfbb377a166448e1223c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702452567/warc/5c11836b15b266a315d53fe9d9be6ab9b60195cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702454815/warc/8fe59b523f67cedb497d417c247c26016523486e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702525329/warc/782dfbd74f645b26c1e99304bd90b6768171b2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702652631/warc/cd2e509ec5ef7771756481a428a12e429a00b734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702718570/warc/1e5fb62730fc31734f63aa5bbd4236492777245f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702730377/warc/8e202ede54fe538a5df5c3938abeed47131881b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702749808/warc/5211f6b8eace635fbe3f66e2df34880a0880b2f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702762497/warc/0d337ad3ff6dbf5fb9beaabeff88b47874677d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702777399/warc/b7912b1d191468800bec03b0b215aedcf5054ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702810651/warc/7e68d90d7d4f28262d9f9bfe7bd5862385c26f82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702849682/warc/58cba7251546e59a2cfd5334a81e83f4a761a97c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702900179/warc/adc404ace4e532b68eac2f696d675c921af2c181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368702957608/warc/1fe269989d23f9e29aa5205668252ca463b32729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703001356/warc/a9c8565a1eb43b62e73d4f74a6f052cebce534cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703035278/warc/58561de696a4ba1d6011d2e7f66ca35843fe7837.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703057881/warc/243dcd8b7bbe37861ca50c5651f8134a7e9ee84d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703108201/warc/33ed1ad4403db9c8f825a1a080638425be086072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703227943/warc/5b5cd01cff4ef51e8257e6e60e135badeaa4cff2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703293367/warc/882b765626e78d5567b908265818a1e26c84a5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703298047/warc/54dadb1c1ec1c40d11a90c2c218c828cd1405d02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703306113/warc/fe2037cf4114fd48854f711d8f9b92f733f2da3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703317384/warc/5a16697086b21b04a894299ef35b0e8f9cd3050b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703326861/warc/501372d56c63e6df04256df59e32d85c45931971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703334458/warc/91051b450c860b0f150e0e2e80537075e33f05f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703489876/warc/92309cc5105d7f1afede6138ac476e27031aaf69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703532372/warc/36407e0699534ab1c8485abee0acc5e04eb85ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703592489/warc/7e617741ce811683ac9c64549d35646da6e19774.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703635016/warc/dddb97a78b383fc2deab5c0aae7572639282202a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703662159/warc/93fe12c4a3052e28bc4c6a06bcde1db70862e961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703682988/warc/a836fd16d7d2ee63ad7c5f55d24b2d1a12e0b825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703728865/warc/2b2c9fb05e69a147282d23467bd8039747e9d6bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703748374/warc/98d5e46b2067c8213e3590c576f151d17b9fe710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703788336/warc/c7f1e22922b4147b6055e0275e17e6a4f1bd024f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368703830643/warc/4677f89f699039c83be6469cef03cc4a2220af95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704007597/warc/7e44962d44dfc43cdfc6053a558fdfc96b85b943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704054863/warc/8d181c34fe46c056a004c6013d9227f319747687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704075359/warc/8b2c60f8a6752597ccfb9df3c296998f3239a8c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704117624/warc/966b33abe46e20872830af0bfd898e298737fdc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704131463/warc/23f0fe872a7fc2236cfc9f59050e83e264098cf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704132298/warc/2084b7dea09bbb735865c7f701a24af92f8ac069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704132729/warc/151f83c2331231e7cc875647a8a3aeb954bd64b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704133142/warc/68dc4706cee08f92d71d4427433a7b91acc5c1d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704134547/warc/c8023cbfe734462d222276a4d0a6826a62ab9613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704179963/warc/145d71e7f2a8fa37eca9f5342b7e6b6778cd1697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704218408/warc/2cad967fe2ddb0de41f5470e9e9d7f5550a39fce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704234586/warc/2407faa96f7182f274aa5a7af5313e5c473d26ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704253666/warc/85a76ae1e4c92a9b9050b1d5e9139c5ece0960cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704288823/warc/6cf59c270c7d58cd8993a64c8f9032f341853d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704368465/warc/85e21737b7cacd06c7192035a2310d7aec9c7f85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704392896/warc/b638e26435d0de0459ce1dcbf3b45a78bc6d4078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704433753/warc/d557dfd3e46cc99faaf500d97f871838e3fbcc1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704517601/warc/72a61632834bad28d1abb9d1ee79054cc11d0948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704590423/warc/09cc875a4e64b55c37c20fea7e76204dec99c697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704645477/warc/42eaf8a62e76562172a13c1820b9707a385db40d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704655626/warc/a07bd2d068d9ac3d4517ccade51f566d203aed33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704658856/warc/dd6812ac1739fd9bf5dc234486b75074227f6f38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704662229/warc/3b139b0cbf670ce42aaa8cfdc692a79fb447cb6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704664826/warc/48f4de5803261cb3e2ac174f5665365c65a3adb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704666482/warc/9d7ca7223b50745165c3efde923904039f51a76e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704713110/warc/589143f0a50045f501878055621ce7790d0abe8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704752145/warc/5ef5d4a2ddc5d01eb979c9142d5124cd0b449b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704818711/warc/5c9b06baaf24aa9d805e0791ffaa95de89b43281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704933573/warc/2f2f1cb9a67057cc7a8a8f70812d6d53e68ff64f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704943681/warc/4763916967ec95c93dc710ad45cceef0f50122c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368704986352/warc/4a5920231d87fe386a56f07fba672f3e1dbfba21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705020058/warc/aa52d25a7992aaddc002d915a82cd9b42f6de1c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705043997/warc/c81941b1d567ae7c4519650f2bc2b27d0151e816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705069221/warc/692cfd9240cea46a032f3038ede7dc0deb25a74f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705097259/warc/bb0e2f60e170384d6a78ee017adb616c56287ff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705195219/warc/f85580f4c680cb7f2ea1ee9f7b849367a0c9a0f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705284037/warc/2143c283c80fc3e6b6d638ae75394ba3cd70faa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705300740/warc/450cbc5eb5385158041acf0eca2464ca04e37105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705305291/warc/9f7aeb5c6433c3dedc5eaadf23279bf18c2f1d1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705310619/warc/fc92b7499384d0f866ebebd2add0e9545ae12728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705318091/warc/7ff4fdf9965290d48ad65c205df295384091ae1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705352205/warc/1f06ab6218b0b31ccee526a66ace8c8f6b37caa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705407338/warc/f6ee0fca6196fdb2741cc379849ed83a76e47835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705502703/warc/b1690f99c1734c5f053ff5ce21cb0e4e8835f85a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705543116/warc/c931f8b271a946c6366538c76dbce6e039ee84e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705559639/warc/414b1ea2263128bd622e13861a0fe0d727f7f151.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705575935/warc/595746f8bf1659a969d69fda4d9bc670d3ef24f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705618968/warc/c3ea49fc12d239003b41413775af93304fb43df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705749262/warc/475cb2ad39365186620f9f88e6dbc9e4fd0bb88e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705790741/warc/254fb7d45b4eac3731a53d16a76db433830dfe11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705884968/warc/202de46ccb521b13f9d0f02747dc760567e1d255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705926946/warc/b0c1683e95e7f4065a7e66624e4461ed03a7d861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705936437/warc/0b539352f920e1a5554743883a37cfcc2faf9dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705939136/warc/9b6039552ff29117d0f4359f55b5c65815d1e1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705948348/warc/486acb0854110aedafb542093df80ed2785218a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705953421/warc/aae1eabe04ef43653738c771b96e15925fe29af8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705955434/warc/9e83a37c20d6d450a99a9844545198c05ec08d87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705956263/warc/420816c38c8c51e4e9153eccdbad50e07e4da792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705956734/warc/7befb1ea00c1029168ab41da8031780de78bd18a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705957380/warc/d1afc7f974660d86ffd482fca084ddbd914518fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705958528/warc/e37a26dd28f000d762272803d89fafa0a1a51b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368705976722/warc/9a1b9bc786f2c34220027198a20d40a8a347973b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706009988/warc/111e7341fac411b1a10ab4901583e1e7f6ac1761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706082529/warc/12078ba314fd3a34c97449396bc91b118624fdc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706121989/warc/953f4e0b1abcdc3e8971a416ca7ffe7e81290ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706153698/warc/f3400d85e7de5bc2ba8739a7ee853d1497cc9e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706298270/warc/42357ad0b4cf48f551fac2b5a2c1a7e5acd798fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706413448/warc/5e6ebfa6837b0694207ffeb14eccb51cc2a9dc4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706469149/warc/6c7af46cff19646fe48c28a63d85ee4c63d47293.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706470197/warc/3e7ed188565f550c762489001305e2568dab3b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706470784/warc/ce5e388cadf7f6882a2155838849cf3ccee618f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706472050/warc/c4ea545c8367dcf3c200df0dd4b59863c9109920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706474776/warc/fcaab14b5dc20957b725f5a624c90825f80578f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706477730/warc/6088377052495a624381da697ec7ca81b0099048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706484194/warc/31bfc4673943f742fb3df5509af4069beb59354f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706499548/warc/cd68405c692dd65a384fc48164c640a38dd510a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706578727/warc/a76df54fb7b59a78be8a5d71c2b695cd737e5f9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706624988/warc/1a83bf3ebc244ddd3cff36dddf8b776eb5b28299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706628306/warc/7a2df1d4be5adbe1de9960ca7ccbecb6522203ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706631378/warc/3dbd41f8db8bd75649348f9a7a7eb0f8f1b02141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706635063/warc/e52aa8d56c753598e4826193d2a19990cdeeac4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706635944/warc/f6a0f9b5ef67ea2533539fa9a11ce4e5eeab6a8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706637439/warc/7cdb7fc237afea9614f7f23cd815604b39181204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706762669/warc/dbb7f35b0dbaf291ad04a3035f1d1aeb36f67e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706794379/warc/0761b1acd3021ff69fe6ea4324df043bd00d7a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706890813/warc/3892fc08848c276318348d6186d55d41a73c9842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706933615/warc/bc378cb0f083c07bfd5468e0a3e4dfdb731e6790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706934574/warc/6805ceabc0851a7c6119659a1e09e59dabdaa771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706961352/warc/b3f83dcdca295bd4b403f1749608bfd5092d11b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368706964363/warc/382a5bd170d31b1ec5bde03f6534a52755b795c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707184996/warc/d211079b25da4736494915cfeff7324ccbf7ad29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707186142/warc/5b338589da756bf943692a90c483623cca0d51e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707187122/warc/495784012d637d2444616dbcec5f0ddb87ed4559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707188217/warc/fe3e5b7faf7903b3a30dc661dd949b9898b01631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707434477/warc/c0bf5a23ddcca991873ec3eaf2f7deb6a4d475a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707435344/warc/ec2f3f481cdf5d5b41339f8dc113f90b74814dc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707436332/warc/dc3fdf6aea2ff0e30282b44c501e5867d8b21f4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707436824/warc/7edccdad3b34aa1dc0cba52c0ca5f786298b224c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707437545/warc/fbc00c6111711d0145e43ffe951328ec456d7b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707439012/warc/6ed0cad7375e2658bc04b46f2ca9926328ee5061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707439689/warc/45a331cdb7f6df7c53ec24b9a86b06385aceb707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707440258/warc/cc7c32de0eeb4367602c9eae7bd181ce0e8afa6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707440693/warc/3e8cd19edb2f32a1ebfb66d333655df00afeee81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707773051/warc/7b4780bb0186422181b32f93e68708b94551de36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368707906397/warc/0395304dd56c04d84b885d01c09fcf72fdfeaa4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708142388/warc/5350e2b48ad07e21e843e0f5b2d7855dc56e31cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708142617/warc/f0c9101e728ae84d6608e4b6891563f725736c5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708143620/warc/fd0c113c6c7752b91377e674960b4c309f77a253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708144156/warc/a1cfd4c2a0be7306ff46774967767ed801452c99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708145189/warc/ba6d431ea04bee51c263b49e927ffe0d8c8941b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708546926/warc/b9b8a9e3642f54f9f745fe0308ca68093fc1f914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708664942/warc/562b9aeb7ad1be65eff632d60e0becf5d322aca0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708690512/warc/a92c85cb4f43c5a74c653e57c942da5d9324a5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708711794/warc/fa6c3e81302cf17bbdfc4e5b4ce5da0258d26bac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708739983/warc/69402ad664586c92b22b8334cae366e579fe0923.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708766848/warc/b1800cf45471a8645a7fabcdd306e2c260e095d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708783242/warc/af1e45d2c7b9da45f47d9257b90aa442522ddd52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708789647/warc/1a4c724aafb951cda6979d59d7983857b7620657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708808740/warc/ccaf424d1ccff7d204eea460d8f31394993a3c7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708808767/warc/3aa2ef24d85d88ffe206dab7e4c558b67c1158fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708835190/warc/16ac20f74d3b0818c3376b2dbd93dcaa1dac0e5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708882773/warc/97a44b4541b59d805a077c9af241bc8685973b58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368708946676/warc/8c6907d1c0a79bd2a9463b22591fbfbd3ca7593e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709000375/warc/5d5773fe87ad68f7dfaa567024582d4ff293d22f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709006458/warc/34ee9a1749eccafe8a409b4c05a18f425a3dac0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709037764/warc/22ea0208468ba30b8ade09ffdda6859aa7f5c994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709101476/warc/a054235bdb77ade276880a55436abe233ea1b607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709135115/warc/4d88079f3beb72370e26bab9cb63ef660d34d5d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709224828/warc/1664226b192abd72eeb53f4de19c5e3c276bdb19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709337609/warc/c3e473e283661c2c6aceca9ac5c6995b27498d81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709379061/warc/54d24dea88ce85d8a84b0d6b80578b84b38ac431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709805610/warc/768ae88c53fb8b941e330c4a0243a6fcee173132.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709906749/warc/2c67d5920c4eb9fdded98d2eb0ded234332e4407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368709947846/warc/7885bff677ea1a101fc58883f827a65d9d07aef2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710006573/warc/46a2ebaa0052fe6e68a3eb883a1f5f1722b06321.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710006682/warc/db3e497ed64b113e88ac8dbd05ced1e5ca966aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710115542/warc/040d64995cf5b65c878555c29b235b8b4917abe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710196013/warc/6ee8ac48aa024989ab3d5ecd4567772ad2fdd077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710274484/warc/437b3a3cd08b3e09f656e823361c785e2f6c4801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710299158/warc/38a1844ca58c47ebbf96a93a2f16ad1759e02cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710313659/warc/8bbeeeb81195cfd8b56d8096e24f956dab21a0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710366143/warc/d07fbf9f77cbd3a7f9590f244f3f6cd37c9f71ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710605589/warc/94987cd17f91899de5a4d2f4b326fa46e67ce280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368710963930/warc/16ebb1a1020a7d2358a683945868f1c9484f7bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711005723/warc/286e93b149ea8901399b5f38b447e9e231d2532e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711005985/warc/86c592625f236155f2c23c462a40883cd99e5f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711240143/warc/75d179a045962431d1895a610175ff5d33c1dcff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711406217/warc/75e08cebc7a15793dfa34b7107b0855df0019f35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711441609/warc/428272d463b941ddd78308df34aed2997deebce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711515185/warc/c7fb04c1977fd4e3aceb4e9d392bacb0de122019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/1368711605892/warc/27d25e67c1fe8cf2f40abf8d93eace45368789ce.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-20 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2013-20/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2013-20/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2013-48.yaml b/configs/cccc/mixer/CC-MAIN-2013-48.yaml new file mode 100644 index 00000000..e36106af --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2013-48.yaml @@ -0,0 +1,628 @@ + +streams: + - name: cccc-CC-MAIN-2013-48 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163035819/warc/f07dd650605c6b8f5639a3b0817ccd81f24bfae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163036037/warc/a8c517a335f1566d07c695b0b8a11837cc9d643c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037167/warc/70b113a6fc1c2f5ad722060abe1eb6591e8f2a27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037418/warc/733b4f54f663fbde8bc05aad368414c4e1aad815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037568/warc/e13a0e4f8c78a1f303ac97f310e8a451e9eb1a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037829/warc/dcc06034155ed5d5736d48ebaba39f671af4ab1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037851/warc/7734b39ecd1d526a5878a2ed60c1d3a6cf10c482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037893/warc/fb41af0d40b7e4c34d29f9e20c409d5c268cd245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037902/warc/6e59be77be5aeba67e78c6450bf063e68ff9c0b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037903/warc/58d1ae906271cf7f019806f279dd9e5701ab95a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163037952/warc/27b99e91be772d371e019c3ac3a69e0f73a736e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163038079/warc/fe8571664db635080c145397d0f883072461ac9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163038307/warc/596c5c544041c226fbcb599e2b45b81606026e5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163038799/warc/c50582e166f135ba78f4ae0313d37d10f03b8f60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163039002/warc/14564be23b06aa78481a3d8cc0029b156312d229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163039753/warc/5abad53915cd4f7697c6b766a8eff7269f9793b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163039773/warc/60fd98f45e92cfa9df6226d1560cf85cbf8f152a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163040002/warc/fdb7f7cd953b46db6334ced2ae4c00d82856b8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163040022/warc/845a6e533d228484c65be6db1b11064afc1d8263.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163040059/warc/2771bf3d0f17dc48867836d4f0b0ab93c016c2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163040130/warc/3ff7d1e9a22fc3ed4f8013a9ff067989afbe8f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163040712/warc/6e9cd04114c4e7779c2a7facc520570b1f47338e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163041297/warc/636002bdcfce22947faf5590cc4b4fe511831db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163041301/warc/64bad7341e09907cf7f81e2081252d022c6670f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163041478/warc/743d87d2b55932efc3d2309eb4e5c9c0c586bb09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163041955/warc/67997ce923125443f913cd8fa31fec454cd5476a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163042403/warc/3edf9f2095d96f5b27bdcec96b66419ad6172fb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163042430/warc/468556c0bb03a5dc402ea5347819e13f9bf92d15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163043081/warc/28cd91a800c56194bd3156ca9e3fafb37856c3bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163043224/warc/cd4743028d966094b26e505063a97ee1cd2266dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163043499/warc/4fcf08b20b5ab2bdf2d5b93f31026cafb6b99bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163044233/warc/4fca9f1854d3f4fd71530e1bc7fb68eb8fbe25e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163044331/warc/7c93180ceaaf891f05d42058a6efbc2d9b3775eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163044524/warc/37fa8bd4bcb1d6dd4cae44c5dc5c46ec62873b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163045140/warc/7c9047975a547676ac4d52dbfef08b0584a1e51b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163045148/warc/23154fc127a307ac42da56154bcefd2cc6045896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163045217/warc/a2df5d335a52cd32d258e87b3f651f18ed9bc334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046049/warc/4222c1729c291a88bd1523628888feaca8d4ff7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046151/warc/dde7fca58d188e147f31ab4294b61e0c88077e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046758/warc/cb78dc426ea7c47c2bb1c30c5707a5714c09d156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046759/warc/d69c20f3ffb364b99efcf5bf8a224f2120c9e892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046799/warc/333844e7dfc187bd643a54ea3fac4c8a28327932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046801/warc/d601c4ee358707c063421f7bb2809e20874a77ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046947/warc/483250c328da26cc6020319860f261b9aca91373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046950/warc/82898610bf7991d89cff157a204a11036f47362d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163046954/warc/455e8c215fb0f93a3ac0ccd5ee2c1dc83e068604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047052/warc/a901481b06b612c62d89afeb64e32767e12992d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047055/warc/90af7c2ce88e7eee34926f2295f0d0639c2a6c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047212/warc/7357dd1161709df4624f4000912505a893455006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047214/warc/9a6978be58aa4ad9a93aad5aca2a31c2cbdd3b4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047523/warc/5e7484a5158c104c60eee2818bf63155d66b42e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047545/warc/9d75e5ab66591a4a28cb9d0b31467bec9d6599c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163047675/warc/f78b5b4ec69117f6d6993b451f8d7244ad3272e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048129/warc/a5c41f990e83507c4f2b27cc295954a6e15b0c67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048362/warc/947c804069a8dc09c8bb0d03fb92aa793e0ea94d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048447/warc/fd51c6050bfe4baf4c8015f00a5e85c419a511b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048614/warc/b844f58cb1b18fcd12b2f42137ebc74fbcc74a2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048663/warc/594b8147ba3bcd87b8cd7698236f6c5205d86a7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048688/warc/d90a789e4f4484ff94b8ebf8475534b964087a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048803/warc/8ae71ae6b586f15ee700f9afea53aee6a0acef4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163048970/warc/4d62eedb1f1b6df09caa1f91668ae26f3f4e0ae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049020/warc/c7445ce0bf82fe0c9154318e4ad03979d1804337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049340/warc/f2b56490ce11af4ea9192b507f861f2f9d8bbad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049455/warc/46b3d22d82b139c6f62455ae77eae03c6b898056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049486/warc/80e0944262c4627d98a2774c5a8f903bd5fcc738.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049570/warc/7a831e13b7e124626ac58d8bcc243f7192722654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049608/warc/88819a6f317da8c2ffafb4e4dffc6f518c9e8c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049615/warc/0c7760b35457550dfefba641030e021a019d2bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049631/warc/fd5366abb47e8c0136b36ed6aa8a1c292420a3ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049635/warc/cde9d9d169c518613780a5028be73702c192aa65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049948/warc/be397cffa4e4dfabc40ec08db2fdb185c822527e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163049967/warc/0eb2b7359415914aaacc2bd2ad3288b7e8833250.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163050081/warc/f3c6ae6e215c587a0089ea80e54c4f236f030281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163050122/warc/01088117b0f0360dec2d71161b33d1870fa7ec6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163050500/warc/0b6822c487638916921c9c7990ccdfabe89fcede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051139/warc/de1d05210cd77da15721db25a1ba450cdfd995fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051140/warc/c96920af7185eda81759348865403a68af9ef105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051244/warc/938e2753e7a6c4cb7ce807ba73b7e4be99189f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051248/warc/91231849b33951f34aeeb3e2ff34b44bc1cf465f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051474/warc/f2758adaca47dcbc73131f226bf4946bc25a8c6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051476/warc/d6dd18af9a3de21a8c916172666aed62a2ee8f64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051509/warc/70a33c9b635716042d83661fe55ba1b449963d29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051516/warc/3f28faba028824dec71387094830d51780393791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051588/warc/0bf3a00588a867e137ab7aa2f52cce32aadc9b92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051590/warc/2abad996072a925236316db38fb2a63c63ef0905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051684/warc/3b6b22dba325acb07d50e172bff739ade7b40aed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051776/warc/4df8a9f8173d09a88ccdbef6a6bbe2353b324804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051789/warc/46a7898eb19f19b57efcdd33c30b8f2dc70b8462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051984/warc/c88d6fd2592138136d1d343be04f6aa55b14e525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051986/warc/0e9844106667987b441b56f0799e39a97170a414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163051992/warc/56633bd5ca649bd068fb159dd406aec005c7c5e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052034/warc/ef7f97b8c1ab442817b59fbc4b7452f2291e353c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052107/warc/d6b197fc998776c4d26da06b3619b41a6677245a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052204/warc/f499443ade697b12dd393041328eb82c32ae8741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052216/warc/34637d1004be441127b8a8f75768bac91293d850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052275/warc/5fe6c2b5d0b94f14f8a1ff8b43918f5e18674cc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052286/warc/70608c02e0d78a5fc4bc31e41c3f7b14ee62fc2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052338/warc/eec80967fd84d67a6217d1c3b2c535ac43efc0d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052343/warc/a8b83873a2e52ec4e04adcfb7001ead7768ff1f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052382/warc/d67df5c81ae2d634599f438ec41d1abb5ae02aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052462/warc/75f95f478442cabf8375f9d5e3cdaee1c798e672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052469/warc/4e5ea969023981932fdfd4f76fa957c7868e3cd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052537/warc/fd893942dad0d09f342b3adca51ef067fe4139f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052593/warc/3b53bbcee1cefa9447a56fbd05aa19dbbed2f6ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052641/warc/ab015eb6e4657282a8f4c020a5f0842671ea0330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052712/warc/d68d18d369c43849b7ef33f0094bf77441df3e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052713/warc/bf4905a9c84c3c729c41bb3e16efdb4d9b744900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052727/warc/129feb738ef7dda0a53ddfbf953de0b4856d0cdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052810/warc/6faba4d4b1fc5d7bc00b538969ac6d3bb0a03c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052909/warc/e9e54b2ed77efedb2bc91b9ff135572dbcfe87c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052912/warc/12f993a976f1945d4ac44c37f9cb2d4e6847bcb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052949/warc/9ec58199a0d3bd7d33e3e2626dada60ab8bbb6e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052970/warc/5b3760965a269adacdc92f7e71c1449c0e13cf76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163052995/warc/370ef37f395ae0794bc21c29524fb013011ce0a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053003/warc/1d565322eb38d4318de801430b8aa205c2e787e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053174/warc/f7558772fb0663bec8f416465e78824f2ef317d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053330/warc/af7d306fa40e88c6ecc7bc6f3e2fad2c5b02a445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053380/warc/89397262e8064f1128e84b02d6daee4fe79b80d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053558/warc/d0f0a57641eef951dbf7661e2e7f85ac09051285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053578/warc/c4be1a652fb619da2d87a4edb9707c0ee71fcb23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053608/warc/c5a260894c9882dbcbdac56f78f394d059a11c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053664/warc/c0a9313c4965995b19a54ef8e2ebfe7b256b4896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053669/warc/9d5bd88d940ccc0462ae67fab420aab202a41bb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053831/warc/bcf3bc3c8f7548a81d6187c67cdd0caeb3d55221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053843/warc/4f7cc161d63eb4a4bcb783de2060eee0860165d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053865/warc/192d078df33824979542d3cb4cf944bfaacd7be1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053883/warc/50ac9af53aad7fe6ab961a69ef2fe37a859d89ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053894/warc/a1b6f4cbcbcaad25cb3ba80f937662454dafa7c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053921/warc/30906d7a591f08ee2071f296ab925de9db221cf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163053923/warc/5cfb0608f40b4677fac9836f85f29a9ddf529d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054000/warc/ed31bca3907ae9e4b5c2dac57cfb9871cdda8f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054096/warc/9c6895eec4d068e93b760b2574182a0aac1c809b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054352/warc/8a1efa0bed95d06638ab3223e39cf68356e5ed6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054353/warc/43dc55c8aa7960505255dd3113d8694895561fc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054424/warc/e25d08454115d48e53de1048a8b05c307bd28f06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054457/warc/d4315ea55fddcbf203f927b2efc3f8fcc451310f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054548/warc/76489d521e6fc8c7991d83355500c7656ff44859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054576/warc/43a48abc76c73b19645d779c8d4fb9c4e35bf7d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054599/warc/180fabdbe6b608b5172bd31c83f70d6266313fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054610/warc/27ae1c0d1192d37dd51cd38e59c3928c132a3dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054867/warc/bc2fd49061ad887bdff2d5e57b705c86fef636c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054973/warc/17957aa7005645ac5dd69b96c36239c8e3c3517b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054974/warc/1309102eb7b785c28bb92c36950ad99436a5e3cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163054976/warc/2ec64939f1826b8f9bb9f4166030d30a1ce532c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055217/warc/54b53abd8838d88fd7c4f3a4836f1d095f370db2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055516/warc/5a071282470723fc566bedaf8adb566160dd75ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055633/warc/10c34c8a15bf1883c3ec91bb5de27f7fc48e3e78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055701/warc/8211f9665e3d1d7b322bf8eaeee710b10bc245f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055759/warc/e8a81d7436087e1cb895df04f07d109d0cd5388e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055782/warc/1e49e2c16f53a7f22336470e2f9c3c44cb3e0721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055810/warc/4efbd76463bb5e15cb320b336396cd150dd4350c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055852/warc/2bd6053d5c85b5874aea1233ece1d16431e715d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055855/warc/107d12a97efa4945790a2c6d9a0fa03c1aadeace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163055862/warc/503364c453a0766da95a671753829c3c4f4c8662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163056101/warc/5f014a7ad9d14e2bb582a00ade97b3c9438e200e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163056120/warc/8fc48770bce83ba32ab5afc0d7af4c3276723c37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163056670/warc/91cdd980356d628bafbd1a0d5d4e0ad31dbc62f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163056995/warc/aa52e4da78c9914e78387147d20913127881a7f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163057146/warc/ebef1ffd88a8a6d9c19d9b44ba9569a6a9ba3f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163057372/warc/12efe0a68f731559eca1ce2a6b59ba02939f553a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163059081/warc/3e7b6aa74941f89a22ed5df324caf67ef0ad28f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163064915/warc/47ff41771efae6116116a8e022f141d25f2694b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065002/warc/9e3775aebf0270793506b492ae615d8b0f364020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065046/warc/960312e8bba4d23940159aa86e655d20e5772ad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065206/warc/87f8dd97eea4f2363ea5fdd1667c5e42c09a782a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065342/warc/176014fd29539c581015bbde730fd665f5bf1879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065409/warc/746dfb577a64100729575ffe508741919f2bedad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065688/warc/8c701aac09dc0b2532e25d9c4f015df68df0632a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065790/warc/78e83f452cc90022da09c10eca7c9ae2b8a195ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065834/warc/6c2e4ed5ed10e72aa73ab569a22efac59cafc126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065880/warc/002c19d0982dc5f9fb32f19ca3ebd96b5939c759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163065934/warc/b703eb9c29559f0f783163e987ac59b8447e3d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163066051/warc/8f9a42ec263243949f22001085505effeb13ce94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163066095/warc/d94778816ab61bd43b17867896ad0c8699077666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163066152/warc/a36877df0c438a50e84cb609b436d50babdd6173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163066263/warc/893ca1c8e4a6958a10253dbeb75c3b4def6e2eae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163066444/warc/e86b40319a19460719766aa05606d03b0b3c6b74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163069032/warc/cd8e278df73344a03d7f1e6737bc922c29b7ecf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163728657/warc/eb2851c39911a5570601f57bdc99ce66a11b6210.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163732500/warc/6d013eb3b1eb2c5e0a8c18a0e3da6a8830d0e725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163785316/warc/1efa879974361456f9dcac968e36520f71f71a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163791972/warc/20da8eb50e716144b2a63e192d4e3af673bf8740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163798888/warc/82c199534afa21976c24c69fa9756d3911a338a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163800358/warc/a7b5506d2225222608057055dcd605a058e643ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163806278/warc/d598f873e28e8efd5c1085b2584033387dc58fdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163809293/warc/4d7353b8942546d09707ad8d01c9a91ab63cd212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163811461/warc/f9268b961c5b8b08fdec752681c19c01252191a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163816314/warc/f64c15489fefce9b6d3cc4f7b7f40965d3e70336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163818502/warc/9349fe4149b1dc30117aec54986ad33aa968343c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163824647/warc/948fc35c6ebcb2d87595002a9e4232c2ae856022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163826391/warc/738a048ceb5f4ad56ff6d897867902611d970229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163828351/warc/c8719579536ec95971d58d8cc4f82041b5885d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163835370/warc/88232045c3893b6a838056c595cf94cc5592c267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163837349/warc/240f12a7cba63164f0b12ec350d25753191f0dae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163837672/warc/1e6b22d0be46d91df0006aa3a602ea820e928b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163839270/warc/7445e9146ea70b465b8e1ee3ba85655763ff2187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163844441/warc/eb293f4d362502720c5632b6c06322ff17e572b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163846391/warc/270c088f24268a6c12493d17bea568174bc6486e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163848048/warc/3113dac581204b20388adfcd5a666f762ed98415.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163857457/warc/eff951579eacfafd61b7530659927c058fced360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163857566/warc/1ee268ef2dbeb1d3006602493ff935ccfb1fa7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163860676/warc/b22e6bb3028ab6f6e89ffae4f643290102270451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163870408/warc/6617da611db49eeb16a6283d024e8ef8cb310d44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163879441/warc/33cec436cc5c5d930411ac6d44a4fde1ae898bbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163890759/warc/e558a5d101e10d184ea0b743cf9b67e12f1f7181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163901500/warc/5ae488bf35f6d9b018b84332128aaf828251c9c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163906438/warc/6357108dc20c8c6e44433ead22550820895871e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163915534/warc/123250839ebeea48214af779e0409301b883e78e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163922753/warc/b4850cfa690d0f7493f6063160bc424984eff7c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163930735/warc/3a8b6a3917722854b73acef0566a054c6e021f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163932627/warc/cc0cb4cbd7bbb56f4d5248461bed86c3700db8e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163933724/warc/89b1e267323176143e656fe8dd22deeea0ac9af8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163936569/warc/740d23979281501547c61c994373963e70edefc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163944066/warc/bd65fb979bd672617a77a47e9e86a6b0b72c89e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163949658/warc/d617235c56015fe810f2cc2f76e7602f06df8228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163952819/warc/9e0645bd83b48174ca2517ad121162ea7621b78c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163954634/warc/823e1d48a82d83be45445ef5ddd7484a6e3eba28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163955638/warc/545374562426fee2b177fb87d61533f65c50003c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163956743/warc/4247edbdc2b6376c9427b417a10a02b606ce795d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163961998/warc/89b1b0f2c3abeda5b7c1fbe02d4e43f07b21c856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163964642/warc/ee59e69e00b9507de907cb1aeec95f58556d30af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163966854/warc/5156f710a47524077c417719dc83b0c157ed360f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163968717/warc/8ccad26a045c694f102cfbaea68d43cdb7080b59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163971635/warc/9d406166336f7a455ffe92e4dfb5c133c1bec772.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163972679/warc/34175ca2a888cd72c7ff93d93e91da2445acb4ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163973624/warc/8c596ec26fb263cf812c6a0acb4c9a6205087f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163976781/warc/9bfa424d94eec979a9fe84ceb57ff41b243fbcd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163977808/warc/10feeffa5b35ee563eb319dae686983ec2cdf355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163982738/warc/b4132f9d0acbc2032d403f6724991ad35c5f536d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163983709/warc/84074a21c60faa003678e457499fd0340ec42f45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163985723/warc/145c2f102249f29f636c3f80b34128fcae3ee196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163986869/warc/3d3a664c92a82ee78fe1ac4d9546a51c1f9455c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163988740/warc/a595cc525e7ba4612d225bff8a9c54149ca1cd42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163990831/warc/5bfc668f3a7d9145b1fe380ebac2449edbae6225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163990989/warc/7b5199662d3ab8144d843a3fca4398998a76cd48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163992191/warc/39a73ab46cf9c1d33185da1ec4637c2652ca3aee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163992799/warc/b0c1dd1898c602d899490999e59dbebf3eabd0af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163994706/warc/80f73e6a147ba9230c594542d6e47686e7b469eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163994768/warc/844143beabaf8c81953e144b632f14a9d173a3d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163995757/warc/8e3b7091573287234f72b0907e0d4ae3d50b52d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163996785/warc/dc787b14cd724935c07e33abcaf1047bc0990e69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163996875/warc/2a6835ef24a6a0d3f228f10b106d4f65eaef5841.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163997135/warc/a522d4b699e35130011dc37a5c7fa534913c14b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163997905/warc/e84f910e19e0a419e6b43bb63692291c4a9a133c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163998145/warc/5bf775a03fa4f696de9a1c249f0b54872e00a495.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163998951/warc/ed4baa17181b0514dc1e6d1a09026fd155f74fb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386163999838/warc/29bb4cf551bc0b33c9d6c54e76a852374cd57da2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164000828/warc/48dd256b487034f13a4e12e16038012583d6ddb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164000853/warc/2438fc4c1bdf2db39d0017dd95e66544fe5a7299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164000905/warc/eb2e7086cc8ae2a915baa1a86fcd6f27cf4bcc88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164001281/warc/d72e97a73a5ab580a3ed368b7af9960edfb0c509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164001959/warc/3f84af50ed28ad9a89094ea8875b3993e979ea2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164002922/warc/a8f2cd6d485beea171fbcda48825ff5115b5e4f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164003787/warc/1bc90e2b052696b1cb9193eb09ef63c9b8ebf3f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164003799/warc/63aea4a275a013f91a4cd7f645939706e78f50de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164004057/warc/f00d4d2925a2c59d03e8c9d906df87e6fbb43f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164004837/warc/2c4c6195362fbaaa117b28cdae74e8a62a464390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164004946/warc/19415e1cc4e59808de5dc727473b173d98d8b97b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164005827/warc/97edfeefaf1033ad4e97046d855a25eeea8d0096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164006791/warc/6a5e1d8b7850d846eceea08fd5762682d11bd5e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164006951/warc/84a74156e202172e1a2a7189c5e1eaec326ed609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164007111/warc/6e6f60f60b3af358899819ea9802581bfcdfd6d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164007955/warc/99bc717ae7c1c39c470af3e5af10c4472ad06056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164009403/warc/8660903fe3b10f655bc00542a171cadada0eb1eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164009872/warc/e80d5dfda563b5616ba6a32a517e7d3934c74e0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164009894/warc/90d49b45efa837677df0e49991ef23538b5de914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164010865/warc/735437805ac2e0d6eb9cf7259fdcda2bfcb13cbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164011314/warc/44872846fdf18689e3663c8751fc67bc2b8e0487.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164011870/warc/d62f6587700219304d04d96aca67a376d0e11a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164012753/warc/b8ce74eff793b42f41da8e8ad2f254eed1312370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164013027/warc/7a201b516273ee95da793cd505a3d40ce69b08de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164013918/warc/e72137784fa42c09ac8b5c1773c5084e4a67d7b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164014017/warc/27538a515036f4705ff9840c924040caf86cc633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164014082/warc/c66a4d35b779d9d82d5629d8b7a9101d2d56d695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164014217/warc/71572bb380a69e29c35572db0510249c56db946a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164014852/warc/fdc55ced187095611b80f59e6c62a74aaf1365f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164014919/warc/6871c709e6a465bc6a17a140dd00906087be3818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164015892/warc/23f8bdfa774b019653b06a6bb430b80b46d58a7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164016462/warc/b5e0c1670c7f2a251d9ef6354b7cf5eaeea117de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164017049/warc/a88d24c55319b349994bdff08dbf2d879521be84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164018116/warc/903edaade497d92934d4e2c9c8cbb7a393dfd9b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164018354/warc/f3b151b11482b9f514bd79849772cf14772f3ed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164018912/warc/9fe65142341cb6bc6f4b3ef2cc318e7f5dc1c86c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164019123/warc/b75ecd6484c05d26b993ce274d9d5ec037116f67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164019268/warc/b6081884a640f5a408a7a5a639db03dae6e8ae54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164019989/warc/3ea271afc4254b3a44981ddd9b426e1cde191ce7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164020217/warc/80734007a22844625f790a192051e23d61443303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164020864/warc/f74e2586664ebb7d5ccaf42c0ff9b890ca5ddeda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164021066/warc/bf9ea6befc476ce155db18e2eea4578eac62190e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164021936/warc/79af491074f4f73f4287898b05d89a33b5c701d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164022163/warc/7af72e734b0c1cef0ec388dffc9a417659f922c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164022208/warc/e3afa2d4a6214ee0174bff56b722a57e5e5a35a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164022328/warc/ab2c52f94601eeba9d300a8c8098302d075834e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164022411/warc/1de9f3ecfae277fe90e203109025162d6c78a9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164022934/warc/45700fab9501b6a5d21a3fe325bea9a237d79fe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164023039/warc/a5949e965bc83ce7a28f9ba47a785d4705c3aa09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164023632/warc/b96ee6ca8d3dd876578828fcc8f3e36833cc7ad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164023947/warc/0f34200e6e21b2696a4c5b83ab2cc16b95e8ea42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164024169/warc/11361031f7882e75c3231ab775f4ff4b57996323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164025146/warc/377344fe5347c949733e0285a585b4f17f357fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164026161/warc/1f8490df5223ffbcf7a36b36530870c7e5fe7ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164026971/warc/493da435e8069f02f61c57cb29d2cd3882fbb6a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164027110/warc/840f07d00a086a7e39f419669e7be34879d578b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164027414/warc/3cd44dd36bacc66e069388fbd3b7eda7b0adb9a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164028001/warc/96b9fddfcf5b195861ae49fa0ea7e53f4573e41a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164029048/warc/3a42d240427df4dbb87cb713c2a9135b7a96f472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164029436/warc/d00010751698ff67caeb5ca67342025f8da4a735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164030159/warc/e1e066788c5d9a654a2916818cfd591995e17647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164030978/warc/af1ea84da19d5f5d1be592af4dd44c20f13285d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164031727/warc/0852e843eba36d64c61463e036e203d1816d37bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164031957/warc/9559e845f21f9deb5faf8ed02e6a2cb8d7247b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164032040/warc/70e49f4ee87c39c81b025ccf12415045a7cbc8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164032243/warc/3ddc342f24f2dcaebc71f10734c884dac87ce9cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164032288/warc/f82bf5716ed2b053404f9f1ee6523b8e3053318b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164032593/warc/fe554f5ce9f553579e3ea5b6bdfc498d39865cbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164033367/warc/c276ce452bee37d566ae3d046dabb55c79e1b812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164033438/warc/a460b1914e95d8f32f167fbb1506d649834b71cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164033639/warc/d85918043483d74763b47bd3209748db42d62095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164033807/warc/968402a91bcc0ab2b654ac58766afecabd1c2731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164033950/warc/b3af64a84aa7352a2f367554dc0a516fa021f85a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164034245/warc/bd5f55b3ef9d33242a009b54a32868bb94522ce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164034375/warc/9dbc68d8a0a4580ed21ad364964b24d868b2becb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164034487/warc/5e261dd2ebaeb752d80f91a754196bfeb21cb2f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164034642/warc/0ac4c50bba215ab05701639756bf37800718c046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164034983/warc/3e9c3cbe1d139730c2fc01e84a993aab1fd06b6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164035309/warc/0551b1e855e6065a2544713e51d9b7c29adb3601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164035500/warc/98a36dbf6269fb3d62d5a240c575871170f3c798.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164036080/warc/8cd4819a8a01a1fd116904f37eb74172c90a0302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164036407/warc/645cdc51fc0f0928c09490ba124a5c86f5fce33e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164036653/warc/ab0a200397523f8c7942af003688231895ed2df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164036943/warc/a4b95c43b8b4c1406a857bac0c49bee4e84af524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164037376/warc/4d88300bcbda3831fbce943b3fc73a5845de756a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164037630/warc/919f3622dfa578a426efe9388f1d410f9002b265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164037762/warc/f467de8d2761ef6274da24f7dc179665ed8b15e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164038376/warc/acad5d54de86d2358e17aaac08eebadc59fb311d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164038538/warc/3980b1db63fc25b8c6fab26979d20a40139fc687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164038825/warc/abaf9dcab469ec0bdf03cb02f35f50451fd1a478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164039245/warc/29098fc415d0bfeddb11ff4c6671f4a8906aed58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164039593/warc/a4c8c9ed9dda005a43bb0e504073d48f175f5c0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164040135/warc/6883f78be7fe914de20c50e1d053f3f69ebce81c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164040899/warc/72755208e1f99e52319611a984856e48c579c335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164041513/warc/b8db3a89e30bb2bf0c2e565f538585f601bd2307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164043130/warc/9885afde2499cb657edfba7ef36e9e01068b388e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164043900/warc/c9b0c1a3b2083292686261e91091c7dd6cad6453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164044626/warc/378e30b916f2cd39128b84be68cc23828266f09c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164044912/warc/4b57f2e0d40592fdea1f31694e38fe7708084527.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164045928/warc/050931f711d619d8cc9ab5c1cca66d70a8aa45ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164046334/warc/5c29dfa701163e197d97bba74174f9fddea0aa50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164047228/warc/0903467605ec35f1b9fa762f129fdf8910949027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164048127/warc/58ba1f143edd07b562ae58755ef03c05730e5875.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164050279/warc/8c2fbd98cc9bb3741c349ca7bce99d5f689a4278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164061354/warc/778d77389f9513470471147c6886165649309ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164069141/warc/98d3ccaa904fad1d3e55eeb4d54e399156e0dca0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164093068/warc/cbaf79d9d3d5c69a5e9ac98c598f2e1032a58d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164099123/warc/afe301daafcd0b87ba96350b1aae41eda145206b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164116508/warc/adc96b2eaf5a17a4bc72c8e3a0375d6a3456c7cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164120234/warc/42288214ca57eff5be22ed570e260c3d104daa4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164128316/warc/b7f66364d056c9283f370104477621b89dbd13e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164330352/warc/a41159baefd2bd118c77c17461745100e46a7d6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164333763/warc/bcc5ef20c3dc1793f7edda8cc3f05925bc9bf999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164346985/warc/eb7c76027d06b92733f4f790686f29bca97504c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164351837/warc/e26b6b40590881a7f1bbb5db24363995a6862133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164359940/warc/c58250823db6987c36aed91c44fdc342703206e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164391000/warc/6d3b1c1afa7573bacdb90baebb32004814d2665e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164447901/warc/622c532e0755c9657d9baf74d790a7c529e1976c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164452243/warc/27eabf2410386f633f040c448d0a30fe40fbaab2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164456039/warc/d2c116848b975c91a005002ef97b9b4f033ecc30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164491055/warc/2cba12018c3a5a6c298a874dbacfb41ccb5fddd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164540183/warc/93a808078fd50b2009ec5553794b2fb7816c85ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164554256/warc/ef3aafa97d5dc0c5aeb7842dd8afa8b5d1128791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164561235/warc/60aaee59c5c5b3b117585749347a18624020f504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164566315/warc/561a7d896be59bd221e40d19c7a2bbdbe1e2724e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164567400/warc/9282780afc74599266644c5d45dadd848d4acb77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164568332/warc/61f60ee89c0122bdbcc35abd7fab47a426988b89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164571932/warc/c7bc3d5f1f4ac0fc070ae1a898fa13dad686383b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164572870/warc/a9903706048832c1c17cf3d083c701952c872ce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164573346/warc/643f6e7e7dcae551b622ce0ebbcd68a9d3acec1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164574548/warc/326ce794570dc566e7066bb7c6d2b5d6587f1245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164575861/warc/c1e353927da9a8a52d3c74483802f66daab7c56d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164579146/warc/00e318bbc4e373b2dd5c6a4c4518eccd89c1fc42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164580231/warc/b3f1dc208892fde4bc42dab24092e5ff75101c28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164580801/warc/57265437432e8dba6855c193c7e09be197f9227a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164580976/warc/99538691f8027d2df58f8e249ed7b5aa70cd2484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164581855/warc/366ab5aabd8247ddd4ff07dbec958dd0ce1465ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164582561/warc/b1dfeabd6ebf2e3b02d4c98b617cfa0cd0cf59a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164583115/warc/e80e7eac58bb009cbcee62cdabacd9713f60a8fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164583265/warc/40906d298cef0033c49affef0416cb596df9c3d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164584555/warc/386a9be8192d77ddb90031fd8fe19a5196b9c030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164587361/warc/188e243a409808430cfa5440812e0c90c9771d41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164607702/warc/e690ebbd41024c4cc75c3729544515c08c901f20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164611566/warc/ab10dad3f014bb6ce4e1aabfe38e4d58819e3330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164641332/warc/f86d8c04370749a77a97f4aec5abdcc266245876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164641982/warc/c6a189554d525fde20e8d93026da1167359277ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164643547/warc/88c2be9ff95bb88b0da3bbc87a0ef95b334d3ff7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164645800/warc/ee545244f520b9699cee17bdb438b74baf03b0de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164646459/warc/152ce6e82719434bcdf060df85f4a0927ac7cdfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164647809/warc/667aaea427a4ceca1c26885c28deaf228ba29a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164653483/warc/b27097d3b0c973bf3d530d65773a6242886bde4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164663335/warc/1e3c1cdb7c341462525aa7b944294bda7de01e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164676172/warc/39db73e9d55b9424bf2e972a35a1d06ee6cc4f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164677368/warc/68ce871b4eff95135d5c0d03ece3b1c90a5a1e4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164692455/warc/1e93c3cbdc8f5a46f93714b74cc1f4c31e62a620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164695251/warc/fa755956e86dc92c1f1700e80de12fd7ed1e7fb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164701395/warc/bd918aafb9ea9378f2e8fae1c02b952abe762709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164705480/warc/918130c0cce9a59d7c0bdcfa464dbbd4176e6a62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164710506/warc/c726ff37e727f880cedb0f1f61d93af3cc1b6158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164722336/warc/ac1bbbd7c49d60cae500da316998590477c3578e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164732064/warc/2614200805f49b22fd6101b8c069599403976b58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164737564/warc/57cc1c667a817217c10404776057acd2d763f697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164740723/warc/7948be92e7d8577ef101622d1f664ab04135d1f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164746201/warc/d308a1e602ed8ab9bf25cbb274c799eefa09b0c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164754111/warc/70fd6561cc38de41050ecd48457e566995bab741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164758033/warc/200314cd32925090176bda0042146051d81e03a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164770786/warc/9c9f37912ca2b67477009fb24089dbe0b494604e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164786099/warc/3c1e7512e5cf99f5ec0f3bfb0a248912e78b37bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164789076/warc/53f3db2dc974436f2774f521ab282eda06ee2515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164796892/warc/787b1143cd5b76a1a293e621e3ae0eb8e3573cbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164805405/warc/94d38ab52b766f2c226b137ce2ff89d5bcfbd8a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164819343/warc/ef3dbc22e53bcb925dbec7e742f543ea85c0c1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164834476/warc/49614256025268516e597359ce349aadb966b4c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164836485/warc/388d760ce11035e0f1cc6793bb3df4bf6f1f85b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164844212/warc/4571f075282df50fd1b44a6c300ab7ab4b711dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164848402/warc/6b83000efd35cb50695729fcd010eb43fc9363f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164858282/warc/7d07260ba7c6ed5678c163d7fb24f169f08c8741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164884560/warc/b1085714ea4410676437d2fce8fec2a8dfcd1111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164886464/warc/e7e5e20f2ac792a044a7c9c195fa9cd1912e6c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164888618/warc/4385e5afbc63401a96623b675a02e847360dea3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164896464/warc/2682577e1b7d8311a6bbc0df528413713f48850d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164903523/warc/0e03151ab06e02d537c7de7ca2a979cb2733c4b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164908494/warc/dbaa13aac18b45d0095a14e228e8a0b35b34db14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164911644/warc/864a12702a9b4f9ecb1d8145da40aff67e1a905d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164919525/warc/dbe2b4e7dabb801bb11fcc1f1a6a9ec410487d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164920374/warc/576f674b7c9c4b379696696809d50ad9adcf16fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164920565/warc/ab9baadd431dab024bb0a0916a784bc301daa591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164921422/warc/00b59ed534954f83e04943c6f61f4d3cb9a6a247.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164923389/warc/d8752dbd898c6ba63e00179b71a77c3d90867320.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164926426/warc/2eddf980e73c399324425a068cdd362a99760730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164928424/warc/551388ae1665e938c8ad3a038cae02dcbb1dce4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164929439/warc/2f75ec21bd7b2933f6936ff087702d34321587c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164931675/warc/42fdc7a216a526bb539edadb32e105d92703ebb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164936474/warc/a3412e231ddc8599aec3aa291e6f5d95203c14cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164937476/warc/cf8620e0d27a66a1563d920a6e0ce27b8f7b27c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164938822/warc/74a72bf84d9157410cda0677b9f68c901f136644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164941522/warc/da5e360cf269ab7404814390b26f87e37e8bcc69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164941653/warc/1482398bf4fd8fc0842a37abc67264e091bb6372.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164943590/warc/4ccab1dab8d3931aef6ec88bc344152d9b290cc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164944725/warc/62bc86b4c04577be9162e1e10fc968f231d02f71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164946521/warc/c4580e0806344fa9a2f03f528fe063ba92f5ed07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164949664/warc/e4446a9abc2f18ac56fbfae60a5aa5682f94f2dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164950517/warc/8386b2c6df3c3101c4064a68ff56591ef162bfc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164954485/warc/0f0047a221cbf17b5a5d6b3c66bdfbb6d21d7531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164959491/warc/b82f63968ea5686a2f15d8e07568212c331039e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164960531/warc/f23c730ef4bd9666aa0c9e2a6c1238630d1d0588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164961715/warc/ae7f7cc05e3bc6839e735232fda3361a6096c37c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164964633/warc/09828658044e6e6846292c4c33a1148c6d6eae02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164965557/warc/58e1b4f06b74c993b54e23ee37e90b94ce18b17d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164972407/warc/1abc808b14a5b3db69d8fd493e8eb0c0b9ef05ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164974477/warc/a69d25e3dfd9ccbd9a35cbac49f21c137f75369c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164977792/warc/6383544fee083ea446be2135694101ed1c566bd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164981569/warc/a176a76be400e5a5290cc77e06e8463806d62309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164987957/warc/a0796304f70751c2a1bea2b6082905a822300018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164989606/warc/8702686a975a8562c386ea858c20b134010d74ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164989714/warc/ff9114024691d90fccff9b1aa3caf195cc0e88b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164992771/warc/cfd7833e3cf2bdb6cc5a9e956c5cbc317acae4b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164997874/warc/226fb6de1cbf3be4d7742d2136fc64bea5f5809d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386164999586/warc/9d9c119f3ee434d453550910a0226340cba8375f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386165000886/warc/67b43d81752384b1b34fd8742e0492add11d93c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386165002851/warc/7322e67489b0460660f16609b523aac3a81690fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1386165158218/warc/3b245ed45b7d2c236424693866d9dbb6b9c54463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345758214/warc/56d909ab9714ca8a33291b1b4b466e54fe255d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345758389/warc/205d6b473fed7ff49c5cb6ef58d8adfae51a25cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345758566/warc/b3dbc25f72a1809eea34df88f83959076fb71ff2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345758904/warc/9ef1d95cd73dff188c0870c4ce9698c3adc5b2a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345759258/warc/93528bac3aa8afe70207244be6ae6a44fd5ef4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345759442/warc/e96c48c9dc1b6cb41059d0dee58750c12c9731a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345760007/warc/a4169ac6f9deb60a1aa113f3be3c94d4197d1977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345760572/warc/67541c2baf4a26ccd768e1c7dcb731f54a266b5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345760669/warc/67d2156a2271a4f0edbd2ee662dc1f5f19f9d342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345761603/warc/5647453755eec550bc3717ad115b467c1e8ae189.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345761938/warc/250e0f5e34656876d47fb91c10d4665044f2170f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345762220/warc/80ea4b84268f3cd632efa45060893ad7688f7492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345762590/warc/7b12aacbce337aca30988dad1ab94217d2bdea49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345762908/warc/4881f350aa27b9b0b1a5c73193153693f79040b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345763533/warc/6a7bb94bb04ccbdd2ff924ecaa2ae15e1eccb704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345764241/warc/acf40c03788d27f0373e5725cc9120fcdb7e8d5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345765796/warc/8c3fa4c89f17f5358b79a2c9487e51105092cf22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345766127/warc/bacf36f6ec544a64944e5c0b19addf300604bede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345767540/warc/3fc96c57f3ac45dfa006185f149790976c4ef334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345768537/warc/1e12ea276d110aa992542eed52936124ddbf0ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345768632/warc/696f56a05e7f9b9795c0603f3c86cc44aacbd3db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345768787/warc/c5e7348ff12b11cc73acd8dc5d106716566cbaf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345768998/warc/46b3d9c683d7bf7738e96ff3649b5210d46469d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345769117/warc/29271e125f0f9c2316108a2d620f35814ff2965f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345769121/warc/02e5a6488c85c103de60f4fd828bb1e837aea84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345771373/warc/954fcef6e424c719b4667fce008c04725f89fff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345771702/warc/f49178dd4c51aa0433d38d874dbf32d4627f96e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345771844/warc/8607a2c0eac27b640e0d4e2c66853144b3df31de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345772708/warc/61a4c14e106b5ca0399c044a01dd871b21c0c43d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345772826/warc/a19cfc07e9835297570c8cd4d1a41ff2dd1c4106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345773090/warc/4b2a150272a6fdedef7c9d7f6d49ceda25cf2bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345773230/warc/f9fc09709de7bcf670e67fe6d125ff87ce540dd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345774311/warc/e49d52007e00f318e3dd4a7faf13416fdc994ab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345774525/warc/b4435ef283e971d1f6a2954908399df1b3a400a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345774929/warc/c437918e4e734711431975f204fbe582f709314e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345775028/warc/7bcab649e4d7a62022ea1c0a8f4ee884e4d7def2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345775355/warc/15e12145da59b0d739cc51551c88c9a8008d4460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345775423/warc/db21aaf86f340093bf0b49e133e307419625bf40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345775580/warc/f4319131b676c11b28d1d334cb698122ebf13025.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345775611/warc/b06104b00ecbbefa3eba82d9303a4f5f98539fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345776257/warc/8624f7f657d997fe97716758e9fab09ae1a27c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345776439/warc/931ae955826876f080284bed8e110c9fb8d07df8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345776444/warc/45697a5a4a42d4e69a6d28ae6df0c2f0dc2d2f27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345776447/warc/b5bdba49cb1e1040eecd98beb3668cc44daf5574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345776833/warc/0f761e681d9572f88c38040d5cd1cbef62b8eb91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345777159/warc/083e02889a18ae60726edf4695a77dac1c4e5ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345777160/warc/df9956f452c0fa299eccd573b8dd006939c31f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345777253/warc/03d24882a28b063f0102683216817ba70f561a36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387345777538/warc/a81ede82a6f6dc13ee5a32268612f93c11c18e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/1387346051826/warc/643d4601fbb2169c6b1bdaa9c8f44f9a74d172f1.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2013-48 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2013-48/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2013-48/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2014-10.yaml b/configs/cccc/mixer/CC-MAIN-2014-10.yaml new file mode 100644 index 00000000..a7e10f84 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2014-10.yaml @@ -0,0 +1,666 @@ + +streams: + - name: cccc-CC-MAIN-2014-10 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999635677/warc/0d33a11130f30dff6e0102fc07fb926381c7134a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999635916/warc/5b83d579c1de3c415928a2f11b18f8e8a3dad89a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636018/warc/340f2f26c368e22af45477a9bd7a619aa2559c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636222/warc/df319ad9241c04deb552db42f6cf4448326309b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636575/warc/15fad4cfa9e45d0ae009f4780a1549e45b3083ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636668/warc/144f66d8f0e4f082f39d558da68eb12e9e465ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636779/warc/6622f831f99820bfa0aa798307e0d048d850c2a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999636902/warc/f5c2369c4e62b99d7612ae6d018c641bb6a09606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999638008/warc/6f95e837a9390799315fde9a7a1cf7d17909981a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999638988/warc/f9cb22cea0382641461b30aaf8d340a92118d58d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999639602/warc/7fca33879f6cba8befe8bb9fb320f9af1650f385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999639954/warc/ce448f8bb2444037362dc8b536388c378b83c5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999640501/warc/222e4cfbccec8dac2688024e81fbd1a4adc7f77c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999640676/warc/a09206d4a80adab987d7370a16050493d4d34cf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999641260/warc/1f0b7b333ab949e7914c0fd3a62ad75dc48f605e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642134/warc/c0fa9bd94a2110fd0c4391f21fb71c1d88564a10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642168/warc/7d39fdccb5195fc8d17791565c7d1e576301a112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642170/warc/f046d6de6df6d92aa15ffb7261fb1ac92fc63c90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642201/warc/2d933ee8cb6439bad08a4a4897479d8e8f9cd29e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642306/warc/a4fe8550c353074a03fbe3847ed8a818413d0fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642307/warc/5bd0ea42c794f51faa7fa2e2d907321d31e2c215.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642517/warc/1a0c5665a190656669db248e7b6cfd5253da9b59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642518/warc/405bd5f2b9e9eb2feea3499ab4fdb8973c65c925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642519/warc/5c442c76d8b7b07bdc97eea6663adcfba0c2378c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642523/warc/816eb5e32115e1c33f168f2a47175d47cd371b25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999642530/warc/09b1f8257e23c144a628f3460a5e93728f0ca272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999643993/warc/707163951ede1f393a11264c4c5a497069f3299a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999644032/warc/1f68bf0c9ad435fcd216dcbb5bfea57e6be6cc99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999644062/warc/36baa4dd1f23291ba6623bbf37ba48fbe4172eb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645327/warc/66fa5fa50496ff11f16246d53ccf447e83da94a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645330/warc/e1d3a9e2c0f4a1e4f9258388a8839d0ba96cb591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645422/warc/1d37a93904c38f390cd0d676a1bbe63f912f1013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645491/warc/26e658b0db2e2c0499f565dcc0aa39584b2e560f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645498/warc/363ab068b54e91267b1260899b6a9624e9823fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999645570/warc/fc94c7a0d3863683690ad5f96735734f02bb80f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999646172/warc/bd507d08da18994b8355501d9a5a51bdea288aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999647312/warc/fca5c8e9a89013f1ae7da3e4cd8edc735b573973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999649814/warc/6a7dfb88ed8800f70a9ea24b9a793b3f69d7c06f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650252/warc/01446b31fc6f5e1cfc3148f5c8f423365092f4d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650254/warc/dceee0d93f80270a72924709124f8854c678b162.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650424/warc/c30ae2dcf9c405f616d04abdb21e1fe8c94c60ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650477/warc/ed12067adbb693936b049a03d9ed8a0cf3d57db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650773/warc/7df0afad2a421e9deb82facaf56c7f127410923f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650775/warc/b32dd288f962651c1ef53dc3c7f1b28f1ab7d6a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650794/warc/e84d25b89c49594293ef93ac0233868d2c36655f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650844/warc/d8d8e8c56c9dbfceae65fa06b1b079b8ecdb4113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999650916/warc/ec5f6b2fd3f199b83fae1f512ed0b1d13c764b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651148/warc/20c02ff8ac46700b33fe576780bfb6e3d23ca5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651159/warc/fbcfd67745e14add1799468d266058a57371a674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651166/warc/53200429ca2f375407e9644055164f8dad296467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651529/warc/b544fc22fc09816a6e050bdc67fbb7f7e288e9d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651577/warc/0903f4cac46941a86fbc9f8367fbf54a70b559fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651631/warc/c9905647f5c80ff91d0e15375da4a4d9b3212d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651825/warc/30e103486a74c945ae45a8f67baf708c92a27faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651896/warc/dd88f1cd285714de1bb1538745b115a1fdf0754b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651905/warc/41a3152384b6927f42f69c615a725d5c1ecc7a0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651907/warc/467c7ac52eaa797b81467e43ffd8bc10b1d0ef53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651908/warc/02645a35c923a5759b7676410e63d6694d63badd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651909/warc/d2378add203a3ce6a6de7a12d8782f77cefdfadd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999651919/warc/5d39ab5ac9431f13d620a7cf0caed2ba264f0bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652570/warc/606ef38da7c2ae7755763b3f510051769b31d1ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652586/warc/da58ba4e8708cc606fd3566404ef6fc606997083.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652621/warc/f1280232e501f3fb54f9f1d5124aed30b34dda48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652862/warc/bc01db3e938b72b42ca96b7f3730b40074d9be66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652865/warc/61a2460e438a361e044daf6b5d5dd100f773fa73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652873/warc/69e8ac1fbcdf3c9aa0993a6b1db7f820fa28053b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652921/warc/ed73665ec862a7f13dfa4830f8665938a823e6ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652934/warc/a9c0147797657bea3cdec3ff32ed69c6652c4aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999652955/warc/323769f261f990c25fd8f37971af5089fb4525f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653077/warc/4472b7259e136460d9494b64b7a138d66ded3cb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653106/warc/6963a85da62b447056e3d6523cdc15a07cf9721b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653202/warc/807a501bc75b522cf24a7dcdf411996d3a4d6cd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653325/warc/96cb37519186f999799bb9b9661cec52d0fec1c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653402/warc/5c8fdd23074bcb1fad23f97b35bd1a635dec9c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653416/warc/eb5cc92e102313999ae4d6644e10e732ab052531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653644/warc/c032f24cf7ccd07aae3db50707cd8040a4d09269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653645/warc/8323ea1d1bdd174e495f4e2fc727148638436c60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653669/warc/f0b49080d5dc5167562f7df5014eee3390913a17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653833/warc/6f05820c3e812264e46805966f9192de14fe7350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653835/warc/74ced6d84de77caed7460b2cd54e34f6aabbd9c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653836/warc/c736914f4156920c7c437b9c5a1148aea72d65d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999653980/warc/f65dafe1d4ae36530a67276ecc8e11f7978e5bb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654003/warc/c314ab33a0a5f055cb7de227dae32df9d938162d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654052/warc/3717db1856b9e69d2b438675cbcd792da0a04f99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654272/warc/2e78c17db5190556f377ed2b5f156bfa64f6a11d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654282/warc/75c8562d55ea2484879839b3f8aab305ccd7e72a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654285/warc/0a0126c2c0aa36b5fc86c8bacc290b42bcdb6b74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654293/warc/472245e6a285e33aee3e7deaf44ad1dd1fe4ea74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654302/warc/62a535e968e263f4e3b63754b86829c61aa3cb25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654315/warc/c90ef075059feafc9e8338d43a10f26d99db4864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654330/warc/d6f6e1b395a08050bc2db322f78d05477808ef6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654345/warc/193b0c6decf8ca48f9301104af204aa94d198068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654390/warc/c2c90176625dc761957190dc7c8c1720289a6e65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654396/warc/20fe0fafea7532272f40387a5cba80d789601a99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654438/warc/9006fc703af05a5b06b7bc952594e2256811bdb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654440/warc/84588efbea012682fa3beeba1055f4b40bba163c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654450/warc/bde6b64593a62392edf09304e49be9a655bc45ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654453/warc/4097efe19838475f2389663ede5bfdc97db782d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654610/warc/2c9ebe2b87847fa7e8f496210bd1558fdedc89c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654613/warc/e81845a203283fa6dd40b9bdb333973da21c7e9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654667/warc/903978307e990a9ea03cf8560d7d4fd1ee882b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654758/warc/e15738ff497ac11c1f227356d50a101acd8981a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654759/warc/a3350040fccab3e2119a99eada66d1ebdd68fe45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654815/warc/ebe342b604fd44a49d27020ecab23db5301f6125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654872/warc/cb80f5de2c705a60da82bdc50b032f92659b0543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999654886/warc/d4f3333900af594d73976d2fe360ec4702f6f1b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655040/warc/757853b1f9aaeae8be13ed4f999dd3a98ba3d03f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655041/warc/1bf4154585deff612bba009f3817df04aac0d82b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655109/warc/1e2336b553944ef8af50a30d5ea032e2dfabac9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655160/warc/1785dd799fb36bc990a8e38b9c658b7bdd115eb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655215/warc/dea724afedb98b74200d855b874d15c515e86572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999655239/warc/2a017a4e991a98655b0bfb1ef4618a02667680ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999656144/warc/bebcd194160bc89d402ce4535b1630bfb97c7a80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999657009/warc/6a01b98dee910fc06d8020b34e234b954cfb5490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999657010/warc/08369495228dcafdaae24cf22d4383577bd90873.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999657012/warc/e6bd506b83a6cf2fff5cac21e44fb3c54fec6e43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999657340/warc/cdda540ac37fd07c2bc574bc170c8fea15e32f49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999659065/warc/b7a9bca877e25a46431e0fa191050959d11fd23e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999661726/warc/b76802748c5de85ce4ef23676d04b171ca820b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999662156/warc/5028dc78d274a1be86890d33e57c1d9a7d7234ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999662979/warc/bb338ec24f6a8d054d97c961543c0f8380775b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999662994/warc/cfb47f4a8f16eac9b822e8e3282ab714f56de341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999663286/warc/789f656c607ec6d1238c742872320321a075c14c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999664114/warc/fff66c34ace8fb024943d91836c406d3b92e670f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999664120/warc/749b39b9e8b2157fa4d6cb9a7e14209d0294affc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999664205/warc/e7b36431657d9231fe7ec39082a80beaf2a9f85f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999664754/warc/ba1be5ab6b3b38819619d2ab09c1fa154647fdf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999665469/warc/2135d8f560d0b53a9942a9e44c2ca45411135c83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999665814/warc/5c672475079914439af1f763631c21a59e324615.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999665917/warc/d1089c6f7c55833594edb50b0ec123ac5c844411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999666921/warc/231a1da13105cc945c1a96b7408d681b20b27dbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999668190/warc/08fcf4b7ecedf020d9de2a8c62c9dad3e9e4be91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999668222/warc/a7a7f9d3ce9f9fa49cbf56655e4685f4d70b5f67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999668224/warc/3c1a52b6845b607361a5f6e8b2393c5dde671ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999668738/warc/53586a40ed7e05729ec930a0e37cb084bd587429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999668865/warc/d31710483629e084cd2adc4310f183b5b1ee6abc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999669324/warc/859906b249a88d00301e6f71e6e2b4f0dcd6b567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999669442/warc/df16d7679f89ca8df3daa3f0eb7d37fe40d36df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999669780/warc/9dd13a3003912700e812ffa4fbe49cc3cd1f9e67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670048/warc/16115b25f0b1eea88bc527ee807ebd0059ed5585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670363/warc/ce27031e90112863d76e7bf50ee27a9be1e9ec73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670669/warc/617a9beab1c05c7612e4bf2f095a8af4ba51fb68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670740/warc/de70ad77081d6ea1640fd69c5250eeccb1d10ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670852/warc/e89af52252da6ac488d19702ca033fb3dae851de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999670924/warc/33b62df18f53089896a943f4bfa6b80be4336b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999671301/warc/221aceef063f8125b181f3af89be4a085f808496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999671474/warc/f8c6514a733925883f80a141be3ae9395b9d560a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999671521/warc/76ed90bb667b1aad54cc20362a866fd0e907393c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999671637/warc/0ed7dfecb882858e6efbfcd33370044081c1f22e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999672215/warc/b26d649b5fe6b27641a8a05744ca14a41de1ac51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999673133/warc/3518488293407a292c151c802b737053d403887f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999673147/warc/66b277bad6b44730de69229afc9f17d458a02e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999673298/warc/b6f07c1d08f7a0f653c8ed1c94b03f4d35be45c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999673608/warc/377a5113d0ded6714eebb00e5a759dbe852e88a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999674031/warc/54d91af39717a113df7dae3402cc9a53105834cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999674095/warc/d707be4a40f903e403a717ca4936b854797e9596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999674642/warc/ca427f9c56c173b0730af0cd0d8b7c4afbf9adb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999674993/warc/17c3f59506b771367c4167a2f7f187504fd2b1e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675037/warc/c8931e099febaba32537bbad327f6e94dbaf6510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675300/warc/5a2ef3ca9143054513000efe885964b03a317be4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675557/warc/b946bf58246d99eece2912f34bdfd53284e543aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675662/warc/84fd716175a081de8b1ae047f5321384054d3b1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675839/warc/711469f62589d87163ad1ee6c029c052c052e81c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675924/warc/4e004d7ecb9364cb97628301dafcf64489c14242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999675992/warc/0e292f82b448aa8b6feb31495d620b6387f99cd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999676149/warc/4f22583734a3575726dd8e5368ab0b66de713e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999676283/warc/5f988147e95ed2454a7f7d61269463b110a71acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999676768/warc/b1c30192ec2db35eeae603a88d318a609acbb26c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999676834/warc/a295521ee09e9908c4d8934557ee6434ced67196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677208/warc/ad36076b91db6b4ec258b57a36acb3f7d9bf0ac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677213/warc/e50611cf4937fab1e49155da324ec52ecdae24e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677352/warc/ea779fccf4ef8ebe40f7b2eb5fdb75ed43cbe929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677441/warc/0a266325f3d878dddc42523e9197d6bbf9e2aa3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677501/warc/9529bd0e84584148d376b711024b6324e1ef36e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677515/warc/9a33cb40ad2f2b0eb8026da860b5f905e38fbf6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677605/warc/fefe03c9f5435464ed0c7b37bb4e45ae6bd17988.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999677941/warc/b9611d1a5d7722a2f9fd455e234373f404c42c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678302/warc/d540d02ec3c1761e1e236022f59817e22804f32f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678381/warc/8dfc092ff0ee10a83205ce4397cade8dd2aee740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678556/warc/20ac2ad6b5377451c201f69b871a68b2e43b50dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678747/warc/65b04ca29cc7ae7b2e9618f6c3ba28292e1d0536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678772/warc/71e60ab9f4f4521677737364e1e6af823e6a8c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999678977/warc/f49c94ac7e39c75e72b3be7b1ed0103c9819d0d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999679121/warc/34dbe8b87b6501b4a0ae8e83f50e829a8f567972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999679204/warc/e528b448896d158565a0a176af44ca85895a0e77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999679206/warc/7d8ade1ad8c81f9e393b3af1033507cb5ff0c1fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999679238/warc/a217a4a0a38b06ee400f57a4df8bf9b6acb5846f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1393999679512/warc/06cc148e5efcf1a741cafd983cfee5136388204f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009669807/warc/62530f0c83013154c003a67c68878ec50341a5ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009777085/warc/87d35026bdf3b5dd6724874d069825ac7aa36535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009829804/warc/6d57eab5b24da29818fff56f8d903ff074035409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009871907/warc/13457712a29b83093232087d546bc1813b355b0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009885896/warc/8b8f7dc78c92a9cf44851aa9426bd3b597d10615.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009885941/warc/8ff8877ea494c268402cda19b03f9c254e292d5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009903935/warc/a83739f0e680cb984d540b67312cc5c6ac9a6b9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394009968414/warc/255a6aa99b9aaf207c92297aa77be9c92a5ec214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010048333/warc/13c40237d2238831e695cbc11939ded3f3156159.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010076008/warc/0c8f6e2656009f9caae7d0ac15b2d9e060a01294.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010115284/warc/fa8c6abfa13e5cb947f07d67f4ad874c03998eeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010128060/warc/7952a73e7108c329d7d0f3f4518f011cdb620121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010213406/warc/ef3d70c81ccd6fe342255acde5f732596df4f6a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010216492/warc/3b61e7872f8de177e7556dd316a838dc0a4bbdad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010292910/warc/48e4b21593f814d264010422dd9f49ca5159f224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010295336/warc/1e403b1b6457862d24fad216d44835a27d4c149a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010303377/warc/7041c2df4f6919c744e4ddd8ebb63fa24aa7cb9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010305626/warc/ba32aa58644e5b4ab424f7ec3434539dc9782163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010342638/warc/c2b306c4f85fd6ccc88c3464814d3f8b4e8a9073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010352519/warc/d79158dc0673b53a2086f771e4a26d5e51314ff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010354479/warc/6be375c9dba2928296bb2707d2a58c4da8965826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010355709/warc/b97b180b97887937b1036c0df4dbcd71636a24e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010359804/warc/4f6d89026495a507176659fd7c6790a7c8ab4fde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010437227/warc/da91d4c8d7eaa2079e3cf1b8d7d9f32ca02d52d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010444750/warc/2a4014cefe5658bd146c05d255b946993e942891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010450813/warc/158aa66bec0840041eae8edfdcd03aab42dbc50c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010451932/warc/3d86e92d79e38f5c3fa1c0cdaf860c3223088678.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010484313/warc/e88b9cc1ee239b270f32b73a6c093664e77f1964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010491371/warc/0a3c5179a90bce02f6711f34c66bb5d52a8c6593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010492020/warc/c1d144c52d40d360ef38fa593cb9b657e978d3bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010500848/warc/d01fbf3ad6682f3f97b22a51e591bd0725069452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010502819/warc/44cef07787472460e2dce39dbfd7053e0768262e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010509865/warc/347f8dd2261c883df38ef43f746a620a1dadb0cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010513091/warc/f24b3544d2ec852c0928536d09d3616029f4890b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010514920/warc/a3bcd1fdb565495d6f1a0ff4843e0cedbb68168e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010527022/warc/b3bf901a178cfb227a2e4f78264e392df2f65f56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010547566/warc/f045920364afdafecd15445f19afe554e44bad01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010549973/warc/cefb13a9d3813ab3f08f2e087625a57bdb647221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010554119/warc/bf49dbd411bae56506838147492f8e0cb4ec0793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010557037/warc/105ac6859851ccb8903584c6c71d77376aa1d638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010557169/warc/407682b6087d1b299d03e0b2ba6d0a944fb3a5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010564986/warc/29f43c2f4cc551615a20d231e9b634026c71abaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010567051/warc/e1ee8c97910510196218d41735e1afd32d631f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010607072/warc/3cb519cc975111b2b666b155dca7541eeb9c685c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010623118/warc/f9e57695d4cdbec474c9b93c39811d67dcb5c1db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010628283/warc/e6ce4aa6d3db639a70f516f73e322966907b7a68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010638293/warc/6a452566ca64ed66d4445f9651dc995537f6d171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010642209/warc/d2823b69cf0e730f7857f0f5ad8502bfee9be7e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010650250/warc/05d7ca4e466581b4a853b6316ee37d9a46cfc4b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010653177/warc/bcc6bd3080192ef8368be8a7c7a6ef2ff567d68b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010657774/warc/da078eecf1c989bc9eca76620a97e314691d4e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010666178/warc/0e42733ef8fdefde67ee9c3774dbf633f20d51b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010672371/warc/025ddb8f104722815517416305ebfdf85557a457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010674226/warc/21218e9c116d5eb996141ca2a02d2db4fcaed762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010683198/warc/2d52c5f6237e32868c5128ebb797986d85f06bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010683244/warc/e0543c76e3fd1e68e32b7271bb6bf419e7ca06ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010693428/warc/02fb9ef218f09fd5b81ed773994d9e84d3b13457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010695334/warc/95f4ada5692221fe4ba6383a6f48a5d342a4dc1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010701848/warc/73290b69ab8bd01ea8f93c1ba76d0cd2bbebd1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010706290/warc/f9970b8a808d51dfd8de003bdc03c5c251e0850e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010707300/warc/b606467e66c14987e4062d8a7e3739357e4d1c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010721184/warc/c8b25d7b7f904005e87507bc8fb482084c90a696.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010722348/warc/ddd918d085534d01f660c3361adfba91f49826d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010732251/warc/d6ec1a951fe17363938399de30710995eccb67fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010740404/warc/f001580b6a0985d90384616bc284e27ef41d5c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010742343/warc/6a28c5dbfb12061f10867af6be349839d8f231f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010745689/warc/508ae8a33d9a756da06b66bc151a44a9ca4000a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010746376/warc/7bad11b23e3cab3360bb6cedc49f408f844ce0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010749774/warc/de6310f1f0a076515ab53d806fb53c751879403e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010758552/warc/469053adcd2e1ae939ddd2e60c1facf6f6d7f7fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010765401/warc/78e3f8d9815b83fd0c570d8bdba2f21d8205bb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010776091/warc/215748c6c8184b0964ea453e999f814846a5c79f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010776308/warc/a8a012207ca3da7e3324bacde9cce41d4189bcfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010779425/warc/bad79c906072933ab25f014b80f056b9a43ec658.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010795590/warc/610b93f904c136d7214a3b8ed99eab7b89382afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010803689/warc/565b0a9d6e9721267a8dfec24a09a045181d185d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010815495/warc/dd6ca35d98819e500fab245a27f8127140577bbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010824518/warc/fd7571d884c5ba3cdcdb5880cd00a11174f3531d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010824553/warc/16b3eec341303c3486c2c04d4710104a2b83bd3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010832640/warc/384f9663f2ad14ada40a4fd8ae00f74c1596a622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010840808/warc/694f3afe6ec3c2ebe5ebb70124e58578d56f5a0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010845496/warc/7bc96c1bd8047b756f15566b68c135118d2e0c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010851505/warc/a3cf20bb8c6066461a5801d2317673604ef28f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010855566/warc/677d31fb35013d47f73cc3a52965d2b0f2bbd7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010856682/warc/2448d6ab73603352dc5bd3912e9127bdc4142e5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010869716/warc/4a87dab9d45958f9e702ec285f9d9c8eb2244b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010883242/warc/316bb4ed18e5563c5fe9d75dfa590772d86a7e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010893879/warc/7d4c5f65c751ad278c97d8af1549abf221053755.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010901252/warc/885fdb7b3a64b642550ba8e4c0fdd7c050cf8c5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010907746/warc/c51b586e27065dcc855cd27cad0d3050ce8de04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010914773/warc/ae1a396cac59edc4a3c8ffe8d2f570d3c77633ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010916587/warc/354805f05e2bf292ecebb37c9566cdb449925b9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010925635/warc/40bcad632bcca1e76feb3645ecf815ea51469204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010929218/warc/ec4e4842b8e64683b901f1e5bb19310060dcab2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010934950/warc/d1f229ee56258b4fd7717efc54ba8f6d3bbbd85a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010951612/warc/f33da3b9d768e9046f9b5f84ff0bed419b084dcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010956215/warc/4756fe6373aebae978709eb263661bfd5289135a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010962725/warc/a6754dab6524f8db3eafaa714df94e96f6a6f2d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010980041/warc/fa88fdcf59a16f091d32df87191df31261b08035.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010990749/warc/9fee0d9f1abc3e24cd48a7b2a68d79c70e3ee655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394010995802/warc/012f57462db425572514cc52f4608e76ac6bc7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011005264/warc/08a206ef393e67e65dea28360674ad97a2b0ba7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011011190/warc/6bbe87fc91d1687621a714a651974d2c07497ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011017001/warc/b566416040d577ee81ae0eb3cf767d43c4640ccd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011020120/warc/81aa8952a67984ed596df7b0878335d26f55aa84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011022670/warc/a33671e98ca50eb955657e728d56b4ffc1ae27ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011025510/warc/00a40392626ebf4d4e53608015b07cca3d26525b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011025965/warc/3d778746e351975e0766ceaf3e9f05a8a0205651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011030107/warc/2323fba942211fab32cfd3a65b33d92903639570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011038777/warc/3667fd54996c0971ff37fa5eebfd5dc274e5885a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011041911/warc/c48d0711626a079877a529e903593540880dab21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011042531/warc/5ae609fad72e0b85613fa942ea21697f52c81079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011044030/warc/0a9832aeb28f2fead9604e20c4bf6e8d5f9d9a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011056097/warc/faa9ad35f1c4b9d17eb781585252ffdae35f67c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011062835/warc/e08b45828aa4edbc39064e83645e935c5fd65f6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011064849/warc/5c2481bce26b4245208460c8d356be5c140d954e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011070356/warc/f7650132b10a15f14561cb97d9d341ca0f40cfe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011074272/warc/2143609b66b65526801962f276707fce44cd0ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011076681/warc/f2f72fbfb1770af9ab0e661203b657cf32955b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011082123/warc/eb339188c32b491504d5cfc905ae64f1d151f4e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011085177/warc/91af2f2d9d4aa9d89bde1b23eea523a6c7d7d0bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011090254/warc/cc96a2d038d911ce705c6a75010db079a2d0a053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011094911/warc/d9701ed61e078da87c4e401f0421815950c130f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011098060/warc/3cd04d05165b9449d068bfc2cf4285e83e2ea8f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011101613/warc/1c82a83ed4e95354c6e831b86b1bc92605fdec6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011103144/warc/3896aefde4e23e154de4eb3468135845a879fe87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011112269/warc/1918e777f5620dbfa9984ce9ecc6eceef08da245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011117323/warc/fc59e87812e8b8f95c9afdcb4ce5e724db233722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011118294/warc/6494f5e7d204a6ac1749065a55024c86405179bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011123461/warc/9bf68e2516ea863f0eaca085031503213218711a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011126320/warc/4738f8a8d5238b9dfe96a6a860e1c81b2485c3cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011126350/warc/e52d439a6c50cc21386434024a379ee46e955d4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011129529/warc/17d4431569a136fd76307234db7756cf087b9a1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011131391/warc/80769351b4a624d07a51609f93fd8b6f6fc3c2be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011134261/warc/071024b5d3dc3432bf641bbbf64fbcf2e7994241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011138462/warc/09883c0e3356dbe38ea79e3123ee96b5a497a4fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011139063/warc/a48d6897d17432d3ee3f2c0985dfe7512a283823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011143926/warc/2054227fecd2be75b959cc86032de89189616939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011149514/warc/61767cebebb14119bc8f9c4ba1f003f3dec81b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011150121/warc/0364ca8ec9161be47bb829de5868c9e7a2ce6707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011151170/warc/f2685eb73857cd5909f6297425505c8cccc4ca4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011155638/warc/429f926ee555e255a8f5eb5ac24a2285d585d40f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011155657/warc/77444eeda72f98852382e3a9447bb985059721ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011159105/warc/54ab67f24fc4d813468800c02810aa2631d96a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011160422/warc/27faea95ed6049340153a0cdce69e813a11b6274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011161070/warc/a1d27d6d47a33e39de0b4307d2005a2c97d9ce28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011162707/warc/85c5464bf454e7a8041e949e5f5f1cbabb345e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011163856/warc/5cf97530270856bc2f357a0fa81674ed8255294f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011167968/warc/2c9176f12dc6b9a146bc00b8a43f7630d7509d09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011168572/warc/ef647ee95da26ed008769d586bb1ed57aa3c2811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011173115/warc/f9888e0c082c6e86358e2e13b5559bc606b42b1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011174089/warc/5d1fee9d2920cab3c1cdcb01fa3502c14847a6b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011176878/warc/b66739426ef2035c3f4812605f8e6c329dab132b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011179079/warc/51173d89ce143fa584397d8274a3a267346b02e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011183468/warc/8fd3ee0074911b82d6a8d335cf4aa50c31c0913a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011184056/warc/7b6721eb2d943de422252175a3ce4b80ceea04a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011188282/warc/8490eab81219056d192ec31bae32b57a1a80fbe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011190529/warc/6f7e25bd9555023803aa9c70501217bba9c2e062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011192582/warc/8bcee8fe8ad422b17448cbb11ca2e3f31ef550a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011198370/warc/3d7aa37a53faafd472634fbf4be48cdafb273977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011198589/warc/7fce9eef627ee2d21c8693c8cddae2dea40af8c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011202213/warc/b573df8abadfed2bd13e883fb942918e11f03628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011205602/warc/5b6459a4e0e8e950ddca322edd9730a5cd8ac4c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011207526/warc/0db7bd5334ea7b5f0a904cd24680cd5c3a72a897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011208420/warc/a061e5cc2b3d8688e9d68a9442bef3d2781134d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011210359/warc/1388b0e94834391507704b23e445da455618e8a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011213270/warc/d3a1382e041acd6c1b7275bf99f694bcc2bc069b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011215089/warc/30aa6f17f7ff3feb7cba2e4fb3dcd34599c36215.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011217448/warc/e30290d6c517cadc71694fbd690ea45c8821035e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011219293/warc/31f9b45ef9bce8a1f80d6769e4e3e25d1bfba914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011220528/warc/7d7605ad61c05a6157b02fc5ec39357573d75209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011221943/warc/ba6bf5f9711894644e188a01c0058800cbf48482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011231453/warc/766fe279717d09d93a0b77cb1c56fabd0d55ff68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011232483/warc/98398795ae2cafb2a2ec37b508aec61e895e8931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011237144/warc/714fbc8a1136700a10ec0b29d7aca40cf17c1114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011237821/warc/a58b4cbca7f331291158116f5549afe7b77c0768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011238390/warc/f297aa4088d54887d6c39e640db10386718ec1e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011239452/warc/943a4f933fc91dbe0f94d6e2dd868619b1877d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011240122/warc/6200826ffa7d2a7053c779622c3421af2004d622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011240269/warc/75b54d4c0b38352a1c1603804c0ffa03495f2c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011240315/warc/0a77c001ef65af28b433de588c935e72fa67ec0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011249133/warc/cfdd08120a172cc46dfb9b4a8ac6bb93c1e837a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011250185/warc/5cc09ed0081788ebe64a3e1142131ca3a3da1263.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011250349/warc/96835e2c070cbef98f91e53611b23b35e0b2bf44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011250577/warc/e68aac0b96127e242dd45eb44d97cff974266dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011263617/warc/1718ff796264ca036f1500a7896b4ac172ed683c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011267211/warc/a8f9407743e6824317c1749759534fa2de8267de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011269626/warc/9cbc5b6add1884b64a311096b1058805f3f4f1f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011278480/warc/4b32ef85ac28ea4c64b938d73736fbaa496f8759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011278661/warc/95884823f3c47f6d12a4f2b9febe6044d472baaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011284846/warc/d29fe94143ac5dcf8fec6c30471bbcc8a74b6196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011294162/warc/3455a40c1d58807d877a8b986f4031207367dfa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011319537/warc/572c9f79c5285e716556a4d90bfe14850c4a0dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011335666/warc/93d985a7195ea0ef1294947369467acde852d922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011338837/warc/e9f6b5936025091eb449c1edba54bb97f3fa898a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011342618/warc/bfe4fd6409690dd331a47a9ac45908df8fae23f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011352596/warc/77450793a2f23a109527c287fca97b9edb25be01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011355201/warc/3a27b002516c778b0137412f22a853d61444b6a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011372778/warc/c503f2dbffe9f18742ba3ba299cf287476b8eb39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011377581/warc/a838bcf6c2871f96e5512c6c49e01316c51104d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011405327/warc/cd4ef065c9f22bfcee9a27ac35aec9a646b514c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394011473737/warc/6f65a5417e87be6aaa04727693c33bb885ce25c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394020561126/warc/b77758a8ba58fe60d331c8bab94f9a1140009f92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394020703260/warc/c1f517e3ceda3f7305798d74c99bec3e54f3bfcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394020792760/warc/0dd4a095dd99675090bc0f7a0b9d1ff5641f5d9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394020986799/warc/92266f1991aa7d9e2485738345f3481e81133673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021083897/warc/a7a7cf85de11991eb4e6b4ccfc9a8ffacca5fafc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021097827/warc/1c6676cea66b33d3d4aff9a83315b325ade84441.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021227262/warc/a70fa875ed9a7239d6abb37ef3433934954c89bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021230991/warc/0a711c1a653731697bc1aea581aca0331a05f3d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021251996/warc/502631b66b12e2e6721c17969b6cb7cbaeb553dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021278114/warc/6c97e5b6a7e4e8d13d6f95f2fde5fee159c56dbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021292989/warc/27e97109b244939084b470bb358230157909714c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021306108/warc/9bdbe2b59e8ad3b468e070c8623a11bf0a74a8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021338216/warc/0971bea1349f90fec252368c4b8d80ec5e06bd7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021342244/warc/94e61b3b99ff5542e9c0f2b30d884b927daa4c57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021365169/warc/b58fe11baa8b5ea0cdf3f9520be73d66d4b8aa04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021378450/warc/0f76b6de3cff48bf90e6c13065f9537ebc4cc0c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021384410/warc/861312e51f394c1d8f0bbcefdddc991cc4931624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021389272/warc/f619b95a146aef2e7582764553db323a289f8d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021400132/warc/6f7fef90e18b9910cc486d8f6b6be018751cd44e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021425440/warc/46395e58ac7adacf7bc5138bedd1e69110aaec4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021429299/warc/8a9d5e751b0595bdafebc38b2577bee744897486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021446249/warc/ab483c4b6fb811b8cbe58dd56a3f26515f7a7da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021453828/warc/bb58037f917e6b0490f92c247b1e3a27327a3a26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021511414/warc/6ed17adae09917f78fc860d3df37530a556077ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021512937/warc/ee465cc4b5077ea643b08060d27a41e7f6a81fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021537160/warc/3b973359fdc80942d8dcaa10096d362c414ab64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021542591/warc/a70e686cdce60ad44bba046f68b3d8ead4d9fd48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021547621/warc/9db2c94693ef99020e8646d5b84163b82aab7954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021585790/warc/409c2f8ec965ae1ad4e98e053e1938e2ff79aee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021586626/warc/aeb7dddd42e801b88081e145d058fe73fe31f1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021587780/warc/2f2a4bcef901e16c707a5e67fba9bcdea22d72b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021643070/warc/604b3d1eb086753c26a5f90be58e0e0be396f236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021675771/warc/f83d26e9afb2c524fb87ff302691f38cb30feb00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021719026/warc/48df1a3212f689723619f8a74d9987aafee1fd33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021721153/warc/1ee00124c22945da1722b6c54d027dc221c3f200.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021724521/warc/9ca027485e49cc68e583feb92baa91c963515fc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021727061/warc/cdc82c528c5e8012836ec9bf843c20be63fb6ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021740974/warc/87d08432db44a75df01548472f843f40756fe6d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021758120/warc/4bee6656e569958dd29e1e560d2d05c828e6e1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021762714/warc/f776314f1be28e5b04375ae9360f43263b529d8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021763647/warc/3a03eabfa5d15cfdc70b05b1cb1f8c96238a1fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021767060/warc/947adfa68c4243a39f5b3fd4510f57fe6d3685c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021767149/warc/ffa59b7878e788edbba47d12f73649c9870ef479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021775214/warc/4a1e2a2d958cdbe85589866276e20a77f36d9629.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021791079/warc/ad821a8e072f965dc1941ea2e94d97b40d81085b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021856395/warc/5d9693d47a64189d8ec7f3e1f5d4cee9e00287de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021866360/warc/dc0755692de39bcb09c8559dde06ee3e8e8eb9bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021872753/warc/b54cb14f5e54701c5835a6702734eb14bdcb3619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021878262/warc/934421f1350ace6461e11d12e7a28f7a18f93a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021889832/warc/6fe1c58c3f48cd7528b32885dc7ba266d0cd0342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021900438/warc/8a4a7b5571020316bdd1fd779f478ba80c8cd6bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021901207/warc/7c5cefa11d86d0629597e4d5c26b76de09940e31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021919409/warc/dcf9dc0456ff65965b4c9167214313f95602b3e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021920399/warc/cab993f2ce2c7f87eb3a3475f04d4d3a95fc91d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394021949508/warc/957e3a5882a24f9000a1b433458531c3600e0971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394022296192/warc/235296d5b51034fb0949bf5b3bd91a35fb2b086b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023065135/warc/bfbc049bd7269277f50c8ed65aa2286e58a0e5a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023122061/warc/118d19e18847a049bbd75bd31c5db22396c58e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023862507/warc/02ae40b1dc194a1db1acf4ff5f888c5dc8c3706b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023862701/warc/ab72247881b13c69f023998790c24b474af46ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023864525/warc/246550fcde3dc0bb63ac1fdededfd18f85525204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023864529/warc/3c53b8e98397ba430ac89e17e5cf34e2e4303099.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023864543/warc/a7160af4efa7de6806da8f821289b5b8c64f87d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023864559/warc/0b6733f9a87099bbe395f81b11e5c78d56280bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023864908/warc/52d596b246d76e58da6e1c1f563a826936988062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023865019/warc/5c1c698cbaa53de4bb87bf30812ce5bbb1c98fdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023865238/warc/c8aefe8bef81870b574da14f4aafde2115e1c5fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394023924839/warc/ddb58e7d7d50c8fedc655abe4cc5f5a4cf2e7246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394024785290/warc/3ac6d9de57336191d9af542d0d5c808d8fe290bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394024785431/warc/ce6265e6a20f5a7e7a37802dd7811b87b080d557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394024787620/warc/3218c69f9fe495b8432bb9c3759082215688d9c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394026201240/warc/c3d804144dce0c1b65115dddbf0d1f619694985b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394026215078/warc/e9e7406524ff697b2e2c81602b3c89711d9690dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678663724/warc/b17c3baae56df199996a5e60b42cdf5beda17556.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678663927/warc/42584314fb45edf73584cef7edce1e3931bd4303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678663943/warc/bfa5348a5b497e595c2e60e6f7445135c411cb8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678664178/warc/029ab6ffb09b92608648b9b4d537db7f36918a8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678666156/warc/b578b597cefd3b40552cfa8110de79c12cfbc1b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678674071/warc/155809154eb5db516dab2c8c02c41e41726e6850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678676855/warc/7b8dbcf7fc832f8da8cef44164363946411afd3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678677514/warc/6810d8363582dcf74a2746a05f290d8853f83398.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678677569/warc/00964e2a1238e3c6e811f9e099e3d9c2bca86428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678677656/warc/13a07933fc2e9ea546c0741fa0714d41e85492c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678678233/warc/9b1d7a0c1b3acf7c94bc562a9fd4c7b6bb4c52c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678680766/warc/c644a5fd8d67a821ca86870485291ece83dc7968.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678682078/warc/2ec0d89fb6686e266e734635a7310aa1b61608cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678682243/warc/10036889955c8a9d6afd6dc834592ce846e194c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678683052/warc/c29d3a7e25269cec6bc8a87fc02714f27f8a5068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678683400/warc/a594c8bbe23ca78b5c28c5131083d89ac45d8619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678683421/warc/c62619d60bc2db6e24d0b76d0650c97657b15efd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678683543/warc/fbf050e7f4658ac594872a9909151c547108e58f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678683789/warc/2d5f264eea83b26835252536c7bcb513f617c21d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678686979/warc/6f3fd147dcbe020fa46d74f127b1418531b135ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678687395/warc/84294afdda419ae5c2039638f66a6ae0b6ae03c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678690318/warc/44e35d45a671de400dcd43cfd8bd3d1cebb82a41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678691890/warc/b5332ea9e29bf368ea05371228ba1ad4caf53dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678692158/warc/7f7358e51bcc2caf895d8624dd786e7745fbe7e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678692742/warc/f9b592913c124692f542b23773139dd57170edf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678692841/warc/d4545adfb277bcae065a289743fedba0e04d5a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678693008/warc/bc6a8497bb92478beea1ca1c8f56d52bec2afe8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678693350/warc/c0a774c45885db9ec4fea02e62640bb6a20a07d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678693548/warc/67333d6597b1d4e0a96ae34b4493c146850e98ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694108/warc/910cdc1791cc04840f3181a300e05b05e07c2ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694248/warc/c28c66ae92bb74219acf0b1fb1a29eb3276a0254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694619/warc/8f2783bbfeba144b6f10f13085cdf7e5fd8c2a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694628/warc/8ada55127832330090b32a52bc315344961db65f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694630/warc/8a9f11e316ebc8ff966f31bb19c19726221e50ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678694885/warc/628a4f0cc590a49412c086804762828d9c0b85ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695499/warc/f1d7fcaff230eb77d45a721b3c0504dc720517cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695509/warc/1d4a1980b25ff56a08bec2da983e426b0fcc196e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695535/warc/9ce6322c51aee957537e41bcbd884ca1525d3c95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695683/warc/0c2a8a5548a594af82867ef20d8948c63501df1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695829/warc/d6d772c5e6c9d5b8b84ea7639b5c46b32912d096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678695896/warc/0c66b0d8937f6a544144ffae331d9212ce2134c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678696015/warc/2b22945ce55c570610fdb63acb09a709c0685561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678696502/warc/697e36f65706387fb01c45d50fb41db098a39553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678696864/warc/8c2515b0dbd514ecf0e3ba0397d561ea03456de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678697363/warc/9ed55ace94bd5830632233068571420ae5838ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678697773/warc/9298a9893097ccdce10c47accfea6725dc2dc27d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678697782/warc/18f448b7c412d4a30ddfb65a34c3d03919b5eadd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678697956/warc/ed60b31828c1812b0a4ebf156fe559a29aba0260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678698356/warc/264d8607897541d73db2a6def488d9cf4bf5c082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678698575/warc/262428f243805b721fd84113871d7842b231ac61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678699073/warc/4d5bb3eb3dd15aa43e47334b03409aedb28b480a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678699096/warc/e00126923da8c14a189cf1c04e9a836e17ffaaec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678699570/warc/3c730797a735d7a1410548f28a6a18f84a735502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678699721/warc/559e4812d5f6d5c1dbbc131fef48468d81a3a6dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678700230/warc/b1bd5ebbe48d1d02a2ea97530963ed66262b3ea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678700701/warc/1fdaf1f64aac690b64795618be00d65d0c66a43a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678700738/warc/019de3311938c7be1b161c51ea2596b779dacaea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678700883/warc/45cf2de7b6f1697920083c755b51bda31c9e3c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678701185/warc/b5c19823136ca095083c393200c3a824b8ccd036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678701207/warc/284ba8eff1b6ad0fd6f77a153dca73cefc91a775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678701804/warc/c798ee4f7255975926e404df99f90a0fc6457da9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702045/warc/7a27b41d5c6105e7870b41f9e3e4c9a287e1f6c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702080/warc/581c2f41790fbdc848a7369eab4f3e2bd60aabb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702159/warc/ed1bf18dbe78cb2e50e5114328e49a42e2925cf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702332/warc/9ee0f4da09d370664aa8a9ee2f64b84143bce14b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702437/warc/c8d6fecaec15a4a223824b01fa696596630bd5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702690/warc/5365b03b81111c3a942d101d92609aa3ba611239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678702919/warc/9d846c0274150fefb1fe3f868620431a8756ed8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703018/warc/0e368dcb87b8779c639d64979009eebae7341941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703030/warc/b895dedc11503b8de878758ac02539bd0ffcb96e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703273/warc/0db506734bd9df3099b6db2fe8e611f24f2137a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703495/warc/ac058a4ddd196036d7f943b7ef7148ea6b208613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703621/warc/c477613784a29725b0044c48b4bea63992d9df7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703748/warc/22e5087313e0f87893694ebc893d8eb471ca3446.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678703964/warc/0549302c73b6051c160f9330ef22c9a795e3f8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704059/warc/7fba24d2082d1efb3141d3924ad25ed19bd202c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704362/warc/02584f42fc4f7e4d2291c15dda3d86f33b7ac97d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704396/warc/3f53e23d141a6263d9fa2a131a4ed9d2ff5aafbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704624/warc/d5f2e7354e10ee71b67acd6ea4cea0b05548399b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704694/warc/637d86721ae6ecbb9e4a9afc09b66b810808c29d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704872/warc/69559cee3c744e23d96cd84510e372d10f03089a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704953/warc/9a07e603c19a4342f2fc5bee137e39fb5068c9c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678704980/warc/2785319b3958400d022e076a6e512d6709abe555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705051/warc/6d83707ca218877651f74ad28d15bea936ea7e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705117/warc/a6dba2f804395df730aaa50f847b5788825b3a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705235/warc/14d6a63c1ad2db1901ac8f6bd89d8410b4ad8883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705611/warc/4015b94ff7969e05c15835ec46a58d0f5f757ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705728/warc/486fdfbbb410bcdced4c85753a1d6c3d8cb02c8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705742/warc/d39b1af34abad8ea23791ccc422b230a26f206b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705763/warc/9fe863a4c14516ac0b55912a713b0c6c8c5f3ae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705768/warc/65e0961086767dcc19326abe00b336a00e8558d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678705901/warc/3fbf51dbbe828cc16eec4d3cf214ba2e5bde98f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678706176/warc/14fa103f0168aac43330ff536da7c458bc56e9cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/1394678706211/warc/641571b92dd95043d223142300e02a9cd3935930.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-10 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-10/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-10/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2014-15.yaml b/configs/cccc/mixer/CC-MAIN-2014-15.yaml new file mode 100644 index 00000000..ff843081 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2014-15.yaml @@ -0,0 +1,574 @@ + +streams: + - name: cccc-CC-MAIN-2014-15 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/0c8044ccb91088b3bb26d1cf604487d580a497ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/0c962080ed10278cd429960736cbba34a4ef82f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/24749ddbe8cce5b3b3db54e66834dcb8634e2933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/82f7fc2771d93a62edec3f826bf10019d1bc0939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/94ccad95a580314dc8b6822b3379f9cc4c51ee3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/bb2c824115ab552866e7fd21c1fbf251c77e00f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/cd50980a411ac2387ee7938c4a76a3cfb9da5075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521512.15/warc/d3f5a7f611f4343bf4fcf9f20632c38177544f94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/6daf5c0f325407a228e94324722db201abbfa119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/860f74f37a5affb538a78ec2854a235962482b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/c04b759698143ef57fda89d7a8d8533785a65e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/ca8a7f280b2e55489c0978d592b21e9e1b44a35b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/cd2b84b66db12481d5692dd2b7a8942538082d71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609521558.37/warc/e7c953fdf6fe8783812a502587d0e0188dbb43e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523265.25/warc/389bef0c4e429bdb726c340fda95930875f352ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523265.25/warc/ba1807f4c436547f6cb24dd7283ccfac12c9f37b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523265.25/warc/e8b1c8574bfc1cfd06a293277d1a5c0cbcc7477c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523265.25/warc/ef88dce71cc7389a3020b4ea832b093fcfa023a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523265.25/warc/f6872d3285698e9eab9ebfb77ce68e8124af5e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/135a3dea6e13d57ef3c9445e50e39c746dda994c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/2ed705cc02c0ac4358830f02a744a02058e920dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/30452c6c4063bef0b6a68b47006dc4b5ff0aa76e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/5b7e61b64081b712991e2de9da883437f85bf810.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/806b774e4e36e56e4af4e24f3e5ca9e74899b492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609523429.20/warc/fe17905fe013469324378351597f1eeba003eb59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/3c04509060172d8d340c622beb7524b72f51490d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/72a2e8404227d2725c1122bd4a67082be52ffde9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/af97593db3febd78780573f636f437953a47a1db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/d3ad15680a3b981efa0de552d31fbf6060fed514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/d576bb540d618f2d8f3c310ee1be054e0d975068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524259.30/warc/d8a33e43a7b47e60350f7b5f01f532b29c995a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524644.38/warc/3026b8875184eabde498c5f3a8dcc693f28b2126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524644.38/warc/416f6be8a33944c096bc8bc0baaea4d87a534d90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524644.38/warc/748b083fe84d3a9c9aa599b60fb076ff1cda01cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524644.38/warc/bf272096dd0fe0349da0fab6e69c2b1f9557fc21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609524644.38/warc/fac3cb7253b2fc634b20883fdc0f0571aaef9408.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/192a8f69cebfd36015e696be7bc43c8dd6732eaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/7fed4c9152447b4ec9fde37de2cbb3b6086c9bed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/8dfc850082b4d07c052a2162a958511a57990d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/94c8abfda6cb94648b0e3583d47b4f12a4eab89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/d5ca1661d7a6080b33febd9bd0980533a83b7f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609525991.2/warc/eef34040c9011f58b6774d450bfa4d66f00399f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/374286b489a370d40ff58602697ee9a6ff986704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/391602e07398b5ae10520042da520b6fc7d0104a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/523e0458c306a21176d56111bcc123eb9dd9f656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/855ba7f6cef1e3b90ebf1b8fa6b27d48ec66d019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/a39178ec4b3a944ea6347e3a55085396ffe5db26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526102.3/warc/c43b27f9ac1b484faf547d59d3914bfc489a0a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526252.40/warc/0af30d0c0d516c6f0f2db7064102659a1adf9f78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526252.40/warc/6855edeb65f4afa0406962bf778a57e09097819c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526252.40/warc/7bfba12159119d57b89c9db0d6bd6ab24978cd43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526252.40/warc/800e78258d2e80817b8723c6586e2201978743f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526252.40/warc/e5a693ca150aa0c7cec930874dece5431c9b45ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/1fe02a955a48447a754efbe52c6b6b5f89ac5a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/70fa9bc64fff650540f89691f2c338f35850a9e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/8395ee7d6f2e03a4de157617be9778d192a34ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/be4755f8b69852a0a4759bdac005b974b990b4a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/ec396b2501c07ef99a40034bc959b2beb0d953fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609526311.33/warc/f0ddc706ff81ca6f6c6cdc4001f7f3d46334c476.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/31100d647642b5cac3459c6cd081d113fba493d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/651ae0cd807754976490387d57e424a819eca3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/92ff4be5c790ae83a21fbbdf90ca1042f83b4092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/d007c43f50254ad142518e29e4a91f07252ff57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/e0de8aa986d543eff5f75b286034583146d3f834.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609527423.39/warc/fbb39d7034648707e4dd65a2d3ac7b9aa31097e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530131.27/warc/129d96a88fa4790372aa17e8fe3b371666822698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530131.27/warc/14954a96e680a8065192b46f074c8ad297757fb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530131.27/warc/6bd83cb1258431c7d05cd9ae1e685627781f09ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530131.27/warc/7720d172c97db279a1fb12efb861108a092c3985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530131.27/warc/9bd1cc655462ed0a3748bd2238c7d7e749f66543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/463bf467c0f8e170c8965919f6c621697ee5a100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/59c6ac155c2ba7ec0c294712dbdc688d8d8a0b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/6a2b0de63ba9a049e8d61caf488cb72ce2dbc584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/72de8c0ec6c15b7e2bed0dbecc4ae73c0547870f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/7878a3aa26e610698357f298d1b0e975c83e1528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530136.5/warc/7c4c1d1eed12d6af2c4b93648ec2c8fc79ada363.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530895.48/warc/04b3c6285aaff9f86cca9aa6da2189004543cc6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530895.48/warc/24c5090eaa454e3bd5d1e70a52809d254b6cb5f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530895.48/warc/a4cc40c5ec0a39aacfd355b7d871b617ebd782ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530895.48/warc/ec610fb29ab9ca663ff3dc524b5d46f26645940f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609530895.48/warc/fe196049ef9c809d54c1f50fea0fce119f9632eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/1bd5bc8b79918b60a3691888aca86b6c4325b894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/4441069a145bdf7d2e95c5c71d88c1ae14e56b05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/6045d09dd85c1bb767af3c2f6cfa1dbbd6093919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/778dea89eb28cb0f06aecba5ba39adec48fa88e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/a737ae4328dfd70fff1bc147fd4b43fa0d9f9732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532128.44/warc/d03ac3b3302f04246b848d4f8d9da5a9c02293ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/1e9f8612ce838825ed58e167e9380a49e4023df3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/2b8a83ccb3e5d843f9d9925361b19a41101ecaed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/4e09e62b7adc9a778909d02afcf490c720219857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/6d1baf910b659ca527f797a2b2bd46ff0523d58a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/81323725b8d1280ea89dfee1ad4d28b23e6ff022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532374.24/warc/c62b125da352b89be8d06c141defbcc8e9bcda8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532480.36/warc/0fb04a0f3936374030af18f82cc625a09ef6ca9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532480.36/warc/36c7e6aeb0b9c31368e1615949e8896e567e3ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532480.36/warc/4709e32718843d0bc5a0dad97bda9397a5a67ac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532480.36/warc/8b64319b4e2dd993b98ecfcda88abe5f2003a5a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532480.36/warc/e9223261e6c6fdfb0c9d7fc3901339700bf354c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/08f60ae27801af588e5324680997d140d2fc1f7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/420043c03efbf894651ac97baa0234f49bda4231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/75a78ab28e33b03605fd107f3e338c4ca0a63326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/af745d02acce0c7afbb34fe826be951d20852b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/b4169ad8a7e98169785a81822c4febc5b17043e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609532573.41/warc/d6a344227810eb1a04ba04ab2ee8bce89f7bcbe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/502d6c1e690069c8d788363340e02f71698f0b04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/5ba476eef4bf0bf0864d013c1422526fbaab771f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/76237080a86de16e3c3f43dbfb5287af8af744c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/ba131cda1630b0b843b9e6d7d99f334aff6c92a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/ca69d3f1716386586fd5e2de22df389f943958e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533121.28/warc/cc3979bafbb6305f9a275488887b98dc64d64d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533308.11/warc/4e3bb31cae673f54b3a67360c8df19b6d7085048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533308.11/warc/66140efc5421a803424b3bae9287be85ac9e4d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533308.11/warc/796a0931c8efa08ebc90712cd986e4ecaf4dbc6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533308.11/warc/bad8faf05e54b0d997779045a264d8a0d3e6f086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533308.11/warc/d212fedf7153e4e0d4308f922cedf2fa9595121d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/1efcd0d650b7524cf792e10a32792e6af7dcfbb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/26842de9134f257ddd4cf77366d8a8cbe225277e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/56fec9b3d222f1c337e8f5d7252c23f33b4b547c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/cf69218cfb6b4c771dcaf36a58d3e78b2771e9ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/d4374885055a1a65e7e80340628f14f3ef4dc380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533689.29/warc/e168db08da11d2e195468e8da5b09669f8a228b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/51b0dd5b25c5d42fc627cdfa396ece2c861487e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/67f235d6fdc863315d68b59b27b3da95ecbe7655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/7b1fb1e8c784cb5b0fe877b3fb91153628364dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/9636ef3d532734102bc40477dc20faf81937fc63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/cbffe4b0f46c7daa85f82f0b1eb2b0e630802ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609533957.14/warc/cd12b7d4de0701beba95f1c18b2313f03826b88d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.7/warc/7becc837894984b24570017ba8ffd65da72f685f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.7/warc/a2e4e24a14f2f5a739f8766d77464c96f0d412bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.7/warc/a9ca3ac6b85618a77be1cb89a564b215c4579b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.7/warc/b3958c5b73d99fcb42626861c08641f761e0a25f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.7/warc/f17f26f6cd6ee9d990846fe082c3eb9dacdfbdca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/125d4d75abf92c68f57b4315d3958d976dfa5be8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/280b4325d55bb36f967da601837ed142b0b0685d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/6e7da7f0d411dc270bced2fec00a1939ce37b789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/6fe07f567d3d64f8221f477001666cf2d1c38622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/b470cbe05b3071b2f7c32732bc8824bbf6d6e6d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535095.9/warc/bffe85370321e4b38e4a7d43bcc6766440ec9f93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/2feb892e7665182d2f0f9f285b7bd0143a419c72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/4208647a03cfa4e42a1591ab0ebdfb3e7b720691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/48a83ae08eefcff37d8abc4987010700bc58f758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/4ff87d1b550b7373b1ca7c98d556fe71d38d3996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/655c2a8627b061893bab972f71cd9a3510bd8e3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535535.6/warc/aaacac9bcc7e2bc781ce3484cc4a7b17f1bde76a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/3436a551f2eea9ad8a5c534a33e529a0ab453844.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/5529a6c86d3f71d8de9f9fcf7b25fcbc3da2db8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/5e6b3ae46fbca5f82f1a944b198d3d442cb726f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/62cb5f5de17c50678485e7ddf1c3942918265d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/9ad568624c89d44ed5e306a0b2e54a60309e56ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535745.0/warc/f43cb405fdf3d2606c44e6298033dfa0d2dba241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/53deff0f0fc515ffbed9eea6207ad5a2842886c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/5e0ff9fd90e13e21b6dea5a00b951b03b54895a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/822fdc19855e7d89be37be5b52429bd3134bd95c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/87d99de6190fd3a1487b0c5f38ed48d7424bed1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/a258f745f09bda93831e4da9bf72eb7275d25bc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609535775.35/warc/d557fee1e7345283ef2da800f61fa8a6f7f72807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609536300.49/warc/6d8ab6b2ac303199c9e63248e6cccfbfbd229e08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609536300.49/warc/80f8c892c4d87c89eb173e1122f14ea930b21ed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609536300.49/warc/a6cf8eab20081ade0f51fdaeaf2574f26f2dadf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609536300.49/warc/a79de8eb6c9cd4e1ee391be0cd37330adaf5d502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609536300.49/warc/f43dba93cf5b268c6ec56e71055777a132aa03a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/013e44ac4f6557c7bcb33517b90438bd9ab3e4c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/11318b06bdf79efcaeb272dff6a6b74b9ab9df98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/6aa23945f2f727d1cba1e576d5eaf62e1bf62ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/7fdd4fcd249a664fbed1801fe3a5f0e5d086f338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/89b12f7c28db3c6ac20b36ad55e7096582e847c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537097.26/warc/daf6453be9588142ba7d46419fbc45c03c26e7e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/3b3176629a6e78dd3fb852968fd853dbfe630ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/4bcb66c5f0f50cde4da6712039bd8730e348c45f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/690ce4b767c570c2d8f64dc0e618558657745684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/a2a2aa3797064edc2cc0d9ad10f2aafd60e3881c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/c62161c3e45beebd47dd7f18c2d4503197768b29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537186.46/warc/e0becf47ac0deb85f8c88fd79519721fdb192200.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537271.8/warc/023924d23f55eaa0511780973bdb91687b42762e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537271.8/warc/133c72be5f0006e0c2af8a20ec25b04186bd5db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537271.8/warc/2b0264f833d83a174b8e593921c59f0d1781d96e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537271.8/warc/489b9a34c5a7065dc767ddbec3a213ed8e1feb17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537271.8/warc/5440d3bef5c20c2f333093c80491a54e914a4ab7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/51c0b541af4cfb39fa14d6cafb66db10da959d21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/965ebdc52993c6bb7e214b05916982f1f143f7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/b3223d4137a7fdeafff7253a65f651a8556254bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/b7e1704dc7be3508c1762ac42bc57ef3da70d6c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/bec95cc260037c9548f1f5fe5dbc4730b8f57ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537308.32/warc/f13eb5176947fa7f5ebdaade971d8d6abd46f3a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/0f43a5df7fa2fafdfaaf6c4d44ae687238801177.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/4461561201e3b0dbdbf9cf7d26f998d2ae10f82e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/702122264bc4ed199c55e86d34012760569c7ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/7a84e4be22f8af5cc6adc654584082c6eab1d49b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/817b2267534f0a113ad3cb3fc9468ff52698c06d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537376.43/warc/b5b2b519a9b9cbf2ec8b70390bd6e16e657aff84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537754.12/warc/110fa7aa7ead46062660da8b427df1f28cef8098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537754.12/warc/271e4fea74164441bf8f1886b3372cd816dc44cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537754.12/warc/52e868874125c7052a220f6a74c02ce11f2b54a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537754.12/warc/5372b86d6483f4ead16194ddd5bae138fd2b11df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537754.12/warc/6767dc5424705e0f2dd5c82525021ea9533fae4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/42de27261d441dacf8674b2e150672fad3af9daf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/5f7db9ef15759bd6916d31fe749b2603e0061767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/65f3e0f974da9ac348ccc5cfca0e876591a8747c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/78d3d1b06ef0d2e2e61a0ef41ac1262ef68ad713.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/a100783bbf4145d025e615aeecfbabb5e357ff3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537804.4/warc/a5194a0e2f619ea6f114eeb33f1ea988eae1ace0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/040f4f64ca43a6fb3cb4917ca061c675d7fcb920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/58bc29fd26637dd4e1ecbe8c09de738858278fa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/89b3eb36c426c2c03f90fa7fa3f8ee36893b61e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/ceb272d2b1b29dea268f911df16ddb52ddcce2b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/d23971ba8ba642a0dde5e3c7d262b0d4777aee5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609537864.21/warc/dae7ff85b26528c8b4121df07c6b5b4cb60e7103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538022.19/warc/8aff9b1f76bbd4ddb15f6e3a5c8b8c3c2ed9933d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538022.19/warc/8b82e1a79d33ab431a090e57785a88d127e30635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538022.19/warc/e13b653d07ff7acfd6e82bcb7186d4a959a09e4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538022.19/warc/e1edb778b95d50f99294f4baeb4afc0b91e7baa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538022.19/warc/eba421ec01e0b5bb345dc3dfe4fb5695b91578d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/3ab32bfe40eb694f568c6e76b25e4db85def61c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/8270810a6ee0165ce6269468abb7584a9ed32089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/8a56db8dc8ae824a4895748778a6c97389213b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/a58cccd5da401a6679820d61fe073c5be482abd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/b8b1f6f83ff3b77af28aebda8b1d086759f47039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538110.1/warc/f4c042593048ee8c26753553482781bb71660553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538423.10/warc/2c34e71da9d9e4210774bdd30991c4443f3ff13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538423.10/warc/7adfb6bc8b69f64e499fbfe70f5a130b4d009abd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538423.10/warc/b28afa731f3bb5711b0ac12a6e014a2d072b2994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538423.10/warc/c664a3d384003b6dc16233db1c8131a9d1f93550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538423.10/warc/fdc48d2cd5ce6e8eecae272061cc9c2de1002ebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/36d7041527d9ec92592d242ffc962f4cad7e4c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/535c9be78fd6fc6b0fdbf9bb9569e2130ea59fd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/7d63a1a68500fb97eb1ae5faecbd6eb18d6ee2c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/b0d9dbe3b6dcaf617dd31caf929478f5685f9fa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/fe11c705ca084760828598f002cd9ebabd94838e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538787.31/warc/ff8d408126e71bea8af7b50777088807ce71b1e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/69d47e2f904897498769af40a08101660f18c25a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/972eac4e28f4c1f65f01c9cc983a7b82dc0d5d78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/a13eff68f49b60724e394a31c48eaca2f22f766a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/affdb06bc379f46cc93dc07d5e3872941722adf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/c462df248642a0786567ce1ec6d51b2dc99e468a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609538824.34/warc/fe20737efb1e65849799f183c3c2266006569d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539066.13/warc/0febe4456da0b57e45b32160dd6cc56015f6f317.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539066.13/warc/5f095128e3bf933e43cd543ca73a1a968c672ec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539066.13/warc/92fe87de4107b35c216ecfecfe071ed66587619e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539066.13/warc/b31aff080fa0fc90f71925620a6899bdf86a0f29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539066.13/warc/c4c5107140d2428d7e153c80d8e0d2dfb894b83f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/115952d34671a374b26d4a5619707c8596d76cdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/6015de326cb41c819dcad351bc0732c4a81aec9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/7b43fa2fad1f4c34e401b4fc252a97a24b1cb1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/87585c3221975fe2e00f9c07b880c34ea5c87287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/9b0536bd4f050a2a3314cb44a0aaca9f2dd03b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539230.18/warc/d0fa3d632c5e99bba308f2c602915f2110968b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/005c52d91a747dd89b6371f49c259d5432d4ab4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/28a802e809fd54b94f4878933621d050512f06e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/892ce3ddbeee068fb38882ee087bdfa27bd2938e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/98c170bf21c1e8893c6e23186e640ff775c78284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/bb912124628bb5676f9fe2b9c96786f1bea74b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539337.22/warc/f9b09fbb5b858f78fd79ecff6c4bdea5dfa6666f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539447.23/warc/2c4b774f9f8eee27b6bad196dfa65d5e68e037b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539447.23/warc/7a1e61e14f69b335a0d9713741bc8d30c2740dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539447.23/warc/86a6ef9418430a086a2fb23dbb6ab4ab57343468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539447.23/warc/c781bded5eed7dde355a328229a2907226b77e91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539447.23/warc/da4f811e206eb7e93e0d62bbb0c751d4e27d57cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/1130cc406c6bbbdf48ed2efab35e5aa81145d39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/4eb88bf69ff317e3596f43539a688de65baa7746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/53a6d28ae03bb1973cf6a8c514a8b7ad72f2f08f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/ca74f94dc19e4e20814c650ce984c17eb043d19a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/f6270a91b8bcfddb8ffd678b56238e17352aa7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539493.17/warc/fd99d83d584e069c283d3fd5c7e2814a7a8787a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/31397753af6eabcf5df1662ed6791a13bd591ff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/3ca0b3cfb6fc396e263d4a4535616712d509cc03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/7625d7e50994c7ce270eba25a08b68d5bc265d18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/8aa66f72a507c5cc30df8cdc3395802dbbbf6fcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/8d69ed019d23ea616a9bad0fc678917f028a220e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539665.16/warc/e13e01468d49c22599c181042a03b689db98c8d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539705.42/warc/4594c21d6b40b10d8a0ab00ac51c9ece886265ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539705.42/warc/5c54546e03f310768ae2e0289dba83945137a1a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539705.42/warc/5d258e53c490ad4e99a9a88b45257190aa18f02d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539705.42/warc/7b203d30556bdd1ff370d4ffdf3f7ddd13e2c77f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539705.42/warc/97b153d1dc9bbe6afe46d642eb69cdc7a3584266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/1d7654608a0416a13cc77450d60fa6988c221a41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/31ed3354a9a72521272b9f60d87a3aed6393a13f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/38d3f179d1e5ea5d821d1e754a945d1745a60574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/5ec40a328fcf11bf01388092d8b4c9585164cace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/72bd1eda23f7f3a1204962e2fc533e56254f644a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609539776.45/warc/922e6cff55b5b0bdff10bca7f0665f01716474b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/8772097f67fa84b19542efe87aa8b586ef0d4f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/8a366cd353bf9c95697b582b1568ac72fb692ca7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/8fa5f02634e6b266bc7636940205660387147561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/9208a7cd54d93efedb743f8c3f8b5f0e71ac3ef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/b571de706f4cb1e20dcf9d4e9a541267867bc8a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1397609540626.47/warc/c838801f9e5fe8529d10a8372bed6c3f65eb6648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/3c60cca8d7835015619ab0540118f9d929bb176a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/7126d6bd9ca46b53e1558dbaaa3c9ef205d2df42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/936ca79c87b6d8df3d6460eec444c384829f3290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/b1b4a79707d0edff4c53af16d7c9da8a92211a82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/cdf6b8cf5378f5242b35d464118620c61cfb3048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223201753.19/warc/dfbd5ca8e8517933cea3388523c8bc9077b117b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/42f4bb312c19526510a67487494a6cccee91a8e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/bf4424504681548a407742a90da2d53fc1859114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/d12288b7029aa308dd78250f44b445508ed82e99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/f26566a3cd07b7be5f5b49ba29dd92bb033fde3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/f5039bf4959b6ff8bf2c8c410f5f649280007f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202457.0/warc/f91f76739be8ccc5ee02709a56b5975105a8d433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202548.14/warc/0c5c572973ff81f810663f696781af779e489d4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202548.14/warc/43480d07a9dda92ebfacab37afb80345273f5379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202548.14/warc/64f79d29b22cbe6be8f278253b12f75bfd039897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202548.14/warc/bd1916844e3b1a3a0cb48469d5c6b914f74c1f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202548.14/warc/c9ca6019fdcd577af9f661b4857e632784e3e799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/0180f8258718c1dabdd12fb59717471697089d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/293263f6976a28a8a68a965d46db00ced09f932e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/4184ebee90b345e5cbf166991fa4f54791920985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/954f8a5c2e30ac0bdadb1fb25b6bc7f0b9ec6da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/a33cf68940f3ab8327911e31d70df2abfcec268a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223202774.3/warc/c991212ff28321fad240879dee263ad2bfbbd4e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/366ec4bee156ff88d82b72251f01e87dc0eb2508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/656555e95fc8482a3126cdf82c06592ad8cd9a90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/ae1f128fec92807881954173ec866cff28c466a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/e16c1b68d61d00e5f3419839b5bff433cdd4331d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/e9b80780da952f3a8c26af5ab7799655b2241fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203235.2/warc/f493711bacae0d6a13363140b9be180bc4a47e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203422.8/warc/7891f636e8c9c420be24ed5d62be36f2dfe4bacb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203422.8/warc/a1d900f8402024bac6c71d36d2966e79b5ad15d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203422.8/warc/b2c6abd978e36f032e8cb53dad4339332239bf06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203422.8/warc/c5eb00856f235d78a4698fdf43776b83400c9f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203422.8/warc/d846e6e822e27353fb5e414cbe6010bc3eaffd83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/589d4c5265358cae2767783060e2af3f578d026b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/590473a63057aa9d9206d4fab89e8b7fc2a87746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/7806f0e1236545619bdf175e42c929d463d4cb42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/785fb4b9b215c33e6d59ef4662b8f2085ce40ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/7d9c1d4c27085c276dec80d40e9c249b1765271b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223203841.5/warc/e3a354d19a91e4a41f679704ef66dfee191c94fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/376c52a9d5dc65d217d1de0a50c9b5df112204d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/3c71332d9bfaab883775fba4e8d6a9645c64cda1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/8d75da306e97ce256f49220b00e8021e18ae5b91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/8f1a8e55a1d427897c5f08cb6bf06558e5f2e950.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/b665043a2d4ac7f3b90d4f1a6ea279cb005754d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223204388.12/warc/d46ee581206f7a6e531162c202a8bb318ef1aad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205137.4/warc/0c0d1cb3c79bf9d9c3f4048dd86b8b724846af3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205137.4/warc/0f97d8ae9b17c729aca1eaa5a417498c486f57ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205137.4/warc/4e11fb6548f060bc43bde525d31c7b9b20f082ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205137.4/warc/9c07b62d7645fe0e91383c6f8f7ab9b20402cc74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205137.4/warc/e07a2bf480527399c53e2b6ecb080e1b080d295f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/5e72e764183116c6f18a854cc7f4139dee2df693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/74e541653646121a607e2272ed1b413b30cff973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/8bbbd5eb9165e5bbc328351654ee0870122d74d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/dcc5d6384bb6de2f64be1da7e257e4568961819e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/ec24d415604638c085ad1a53519ee2b6f87e489c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223205375.6/warc/ef2c5685991e7e15a221f61198ec5f8ce66b6992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/07cc1e284ca6f6ec709b49a45c98d788581f79c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/0e38582466ba03849a062d4d6633b21bc7967e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/5c7d60694b7f9a2c63309321f1c3be57a0c4edff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/948eb4b96609a18072eccda9e715882d83f813ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/b7afd65c95c36de06b417ef43a7b7157fe209913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206118.10/warc/fd88449824cf0720bc94bf8a33efda0874d87820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206120.9/warc/2b23b02f853bf2ef1ea1ad912d5d1b5245c9c91a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206120.9/warc/438d66759f0242fb766cf7bd0543cb3e564edb28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206120.9/warc/5d4998b72d4883b874e353a004e7ab7aefb29887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206120.9/warc/c9d753698d38927ac2cac7341fb92aae5119e444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206120.9/warc/e013dbe2a6b36ac9b48d9d8249a680139240534c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/ac7b40279bd48f23462f8cb2a1176d093a2b6192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/b4112647477bd0d94116977a05babd2ef1a9fefc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/d188d63f545b18d61fb5cfa4e0845f4821f68d7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/e536d8c7658059a4af308c1c1fdfc16e03ac49d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/e805d54ccb618b5dcc396de4aa0cb412b2be492c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206147.1/warc/f032acff2074407550e592c37a4d27b2d7222ff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206647.11/warc/3b73c075821e74ec2b5f750e4feb6e3f2b9d15c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206647.11/warc/6a2151812cdd3e5c61966310c92f73c24f5fedd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206647.11/warc/6d9a92585a700c3fd79548f787746b979b006131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206647.11/warc/954b7793114f9920b42ada01bc85c548374b3636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206647.11/warc/a6b847b6983203ac3e6879752fdfa443310c556b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/10a4eff1d00eae558d147509cb20e626e44cdbe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/7b09d71173e3344b6b2620f819202836faf70c47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/7d3c6248104364a741215b66fedf97c4fa9e1862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/9a2b62fadfa7ccaca7a1575de59db8737aa1dc53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/b8bbf01a277687a33ba74ef637b2655f02d162d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206672.15/warc/e9a1e9d91c0df0e918d2039a04817f988576b196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/15391c49da50c77f1ecf1fd8b65cc1d18e931aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/b11d8e8c15caa9806730787806c9e222fd7e347e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/cee19c730369b5fc87b9eadc7502e1aa88c0acd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/dbdbe61691bbfcfa4a2e54e2cf190651749724cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/dc3c8ca91e88a5006cef3ec32361e884f83d1587.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223206770.7/warc/e4e7822381a7fd23d331f740c0c02f6f9195c941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207046.13/warc/104580de0e8789449e323f0a49e4ba2971097a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207046.13/warc/2d9efe04b7fe66e3a2b4ba2bcae6fa11782ce062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207046.13/warc/96da3cbebc57ee57c754e64ab9d556094c772ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207046.13/warc/bdd101b8d26a882198d9a8ae3f490d5b4a1682e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207046.13/warc/f1fd0f1d5dab2aaa575c04e6e6488a998b70b80f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/23d33a5db53b7f4c53039e97e110e3c82a9ef8b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/52a6c4df1200ed8c5d248493c4936893ee408b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/bfda8c683f4dee08e5d2dd2a52e1b271be7cfacc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/c3675eac21849b024ad4ac0b8a4667431782af95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/c919b661636d11806cbd3435e3dc7584b7507ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223207985.17/warc/ee9b791f60d3f602ceeebfb5a04c25745f9a35e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/0f87f63efaced70bf039cf74725396bcd85e595c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/4c32c614801bf5b68228464962ef0c104cd305c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/6e0f66de14671e7b502202952883f427646675db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/964dfe301b5b0b74d10fcc3db095d748186f22e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/abdd6a911b0d052cff387b576caa08738e2b9a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223210034.18/warc/f799c9adbb6b042a348a28212eb4af9e5a4ecdb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/1442febeaa4a786eac7ba92f048823ff836cc43a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/34cb6f2991dea5f309dc48dd49554aee26fb8fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/48ee3d1f296539f9df6437dd8b693fe5bde5601d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/5a582835eda1d58f7b0a9813ab34ed940f5dc5cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/d4b0c3c47272210e19c30fa44e0f8951658bb695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1398223211700.16/warc/db0375d88bf8784e6b12dacbef444116b6e86276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/13a4869b0b5d7375158d916b259b71a766655748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/16c00660f4bc5b4b18c5f4f5ce10e40b1d0971a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/1abfe1293bc3f253ee1f885d60862ec63e72c153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/2174e0f2df23053164403e3c66415c4f0e161966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/227c140052f660772eac4ff68410d04439b93e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/23d5642cfa96b01c7a786251f697c56be5ca8a5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/2414a66b10c812207fe9d2468acdf6dd0ba19d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/29de245a746572c861a3340aaabf7ef4620371df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/326d4b042a4e64afffd6ce006c348c9134755e5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/33076f080443e16442c94e318ea978e7e29618e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/3424f5e63761bdc9ea5230ba9407210c94fa0454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/367f99e4bfd6e44658f8d0dda054c6f6c21e6d24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/3b18a115232f4fb3691ab0ecaa08be2aac4baa36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/3d8f4bf59fb47dc9bf904da9a60506f89459f606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/3fa6ce4108d2f8b773970d0f8bb1785a551b91e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/43bb1dc07f8b9996e3700cc979b0543fef095ed6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/447843e0df94bb943b55111301fc0870021703af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/489973e2ee276c1592a68dc3663c1fad1089f5da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/4a2439c788cd33fadcbe93acb26f8662f1a5d758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/4acbe66ac9e61cf04db4745f3ddb7bc31fdf5d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/4b0f72b143709805bc540403bc9e6a67aeef0e66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/4b58a2f50ef1ea4d2b54547e7bc81ab469f3248b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/4da7c21ff79990ff6751cd21b4ed55f8585fca48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/53d19e7cac8b02a326967c70ebdb4264563f7a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/55ac55438a6bd4aac3cb3a5b7d9222f699c03542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/57848cf16aefdb1e90391a7906f8329c9d8d6134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/59d8c2f83825ffca51ecf7f32a16c0b191ad86bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/5b02195b744f7b6d9d337fa263984b8980946e6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/5fda67768aa089c42ef8d7be8540ec91b23924a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/66c3520466edc18db8c14e6e72b7dcc671b968d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/672f6d4c1bbb8311411a69d3e6d49a881ed33d0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/6a5525587d352dccaad55dba3c82b216bbc0dfa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/6cb072a00145efa973592cbe4e3c4360a6e5947a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/6edf58f4e509d0365522eed007daeed1967e628d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/7582814d760baaaa6540d0d9eba6748264120300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/805162259ad3601314b2c80bf4498c2abeb13302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/88425f901e9ef679d6a310baa62936bc90face02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/8e9f43bfb9405a717540049954120b498e68d4ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/9512e48b9d7a872416699676b93a7708bd3b5c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/9948d860fa3c05676d125480395a3cfd13af33c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/9d747cd1ab734ff0918f0a087d1613def9fc0242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/9ff95a95a5b25d808fbab2eecee7708744586c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/a13aa5d139735f0ee9ae7d0f3862958624398dd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/a32347ecadc44ce8a945c85d8830b828ea9cd8cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/a4b84e5eebccee73923a7531b7ce3c17720cc144.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/ab8a4bf8daaea4f028509c0e9dc97fd53042e922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/aba2096d2672d64affa364328e0b1f16fa73f3c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/ac284c28ad78fe1f7f50bc42b286d10842f1cd3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/acb1b3f2631de480a2328db3b6519f6619009f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/acbb06a18a2523289dd23c0b95618413a3301e31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/af802d7e8cff890fc2b10b286cf3328d1dcd108e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/b4972cb7334e55482b2401af1ea433f554c652ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/b66625ab9cdd4e64fae81f9b168b58f480c213da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/bd6257d9be58c6c2a7acabfee80172a5ef53dff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/bea0133e03dace47d02198ec7e78a9fe8f7a0724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/c8e721db8b111fd6bd28516ee38f9f55d0a9035b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/cee644a8a3102417db7dddd9c3c2e655e6bec15f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/d0ad6489e37509cc723ea667a3c202da3c3ea51b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/d84f86ffb135ba7de6f119a00d25fd52b6065e46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/d93ffea777be04be929afd1bc4aedcb2c45843be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/ea41bab5c94463d7b3c51c587045f593bad4ee75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/ecc66985b17ff02e701a08502fe1609cf7b9cd53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/f092b3f80f9a8af1e4844ef06f499510b9538999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/f184d09f1cd867c08b5f603159132ab249a58e06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/fd1e1045e6c57ac449b05578f26de1ba5769bc08.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-15 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-15/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-15/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2014-23.yaml b/configs/cccc/mixer/CC-MAIN-2014-23.yaml new file mode 100644 index 00000000..43b6da67 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2014-23.yaml @@ -0,0 +1,745 @@ + +streams: + - name: cccc-CC-MAIN-2014-23 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/015e2f39115fec71e47da0eba52936304ed1c87b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/01b6a13c684d379145994dfdbc05b2f14821e438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/031df5e27a58423084b0e5a420635d5d0c2b0067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/03d921777802962aa1eed763364187511f0465bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/04efa16d74b25973bdf082498d6e4cae67407b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/0718f4d844d02b32748499467bd29e664bcde9b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/07ff2f3c368ca01f4ddff90bc88a2b596e6a342b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/09957787cf46808f2df368cc3ef78f789e2368eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/0a8c252113a2fdafcbde0490210d66d4f3b6afe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/0c858d5e384d1fef58c3988ddfe56f332a7ca341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/12ff01bf30156f6ae84e64fff6980cec85420eb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776400583.60/warc/588c1565536a8fb93d81ba0291b3e753cf78235f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776400808.24/warc/198a49335f6b5f42dc7a832590cf017ece6fab2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776401292.48/warc/b541e34398656a1f10fba53b48e761770c48ac92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776401658.57/warc/d31c1d2127f18981cc603cb23da033caa676119e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776401705.59/warc/e6c5d2f37f2b9c42387d12f44c4a6506c00574f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776404630.61/warc/4e4cc8227855d5b2439a1a54c5fa584d94de4b5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776405824.29/warc/d177ff23192f9bcc633d2ddcad342d359f991cd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776407319.89/warc/24ec32b87f81e6ab3a4b45c1ff33b40aee39c4db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776413052.23/warc/d86f1b2fadafb211953046298ed7d61b4757eecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776415016.1/warc/c2b57818241d66d13783ed40f43e0a4544b0812d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776416489.79/warc/f1bf7781c4612ab4eb33612659964e01985ba844.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776417380.9/warc/80b2465d0afa5e17802f1666865827fa3b7c3cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776419693.66/warc/a3f68f4de4efebcaf92f5761947a74a41b45de67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776420123.16/warc/10af17a1308856ec2f65985bebf91549ddce9328.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776420526.72/warc/bcdc7fa4d5c99d4c167a6de5d5d52d7512a523a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776420978.98/warc/5c2ec471d9882ddf0cbfa59fe760cd814f95238e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776421646.0/warc/cf24fe742dc831cf566844c6b5f3abba14b77107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776421879.69/warc/daa68e7c2f856511574e9421840a408d1a0f4ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776422137.22/warc/3ac4aa394b2e13ac332bdd6f7a36156411448dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776422751.47/warc/0df3d67fd0d94bd5dd6a5f13c1464053788d7bd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776423075.13/warc/31d8c065b67c389d7ed5854ffe7b23b75379a3da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776423684.4/warc/7fe26e4240f0972d26c3450228fa2fc756de3a7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776424328.95/warc/d870450d95a8d102351bf8ceb3102ae58dce168c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776424634.96/warc/9122c6168167435da1381505e9b70dae723232e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776425157.62/warc/d460621bfb71424538d8727c836ab34f2f939b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776425626.86/warc/2349df8535fcde67e183a5d2324c3a67ed85d653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776425666.11/warc/247e27757ee44897fae79ebd50fad34dcde8817e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776426171.91/warc/a4bd065deb5d34eeb754ba1f7c74889355a37462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776426419.6/warc/729aeec51b7e286100d0091bd62f2154eee1212b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776426486.74/warc/d357d52ca07e2ace67e2f1eeabed0215d2cebf8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776426734.39/warc/004cd7d4109a8f72851b221ef008ec4ae6346646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776426922.10/warc/de3e4467fd81b309d112b8e5e5cc4b04dbc8811d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776427226.68/warc/1e10dafd844ce99fe1bc0d2753bbc5886b1276cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776427481.71/warc/54d90efcd97679a81f61f1383f59913d18d3ceda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776427615.55/warc/4b1ee429579f6df29bceb76b6dab828440ab9dfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776428273.44/warc/8514f784370efa0244d203e16da1463f49433075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776428349.3/warc/ae1197968f5b5ec2f67c903139e504f8f7ee4af9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776428735.82/warc/b3fa9834a5539f6b42eb12bd3fa55434bb101760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776428772.70/warc/d19f92c27438afdb5b13c85baf20f649c281bdb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776429391.31/warc/68528743acef91e00e2944522e83993592a77cbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776429773.19/warc/4634a03db2f5fb0d1eaa6438d937d0b387145081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776429991.81/warc/2a2e704c3d14a842f23d1503f00039f5f9d99ea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776430044.46/warc/c8d527db2a5773a63224eaf680c4cca6354344f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776430107.88/warc/0d5863e54e6721fd4973303a82e0bb3c02635805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776430645.99/warc/bb5fc2d2c63b7d1dea7d57d214501f919b2baf8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776431056.5/warc/b642151331c834558dea19187f9b9d1023508bbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776431152.42/warc/17c954369f7692049fc24333114a6bd4c424853d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776431372.37/warc/ed29788b5d40bad9929e6b1bf0ae09731c14bde9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432195.33/warc/38c1a4e80c0230a2cc898459f0473b4a3f083640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432786.7/warc/786010091366e8ebb98da4c9bb9502adca546e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432860.32/warc/f112216ce8a29d5508498b52f6d3e276e915bbbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432874.14/warc/22d3a51dbe51e9d7fe6c1746f9556e859b4adf67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432893.52/warc/2a896652344dd8006d2853aa6e72a4fa8ca002be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776432978.12/warc/bef866c9f3539f663aa9a084b473ce17ab6c1f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776433958.15/warc/17aa7b9c6a9b82fc0fd68eda3ae62e745fba2840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776434088.75/warc/b957cc6588e9a99540a63eb47807cd0436a61f01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776434099.83/warc/25e67546c84c55b135f8baeeec92adf096cf8b8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776434179.26/warc/4b7b9f0272d3262a56f0ffa02cf0017410cff49c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776434475.94/warc/1d48ad160531ed0ed5d455d0fa02c1f106ab7c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435102.41/warc/d53116f723b9ce02031d37c29474316dab8f20eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435439.76/warc/d882e2abf8ee5e231a4ca7bd6c694e0eca07856c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435465.20/warc/f39884c5d91e6800bb4eef3c7ceb4cd8959b9cac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435471.2/warc/20418ae8193a5d719bdf6fb748e111d2d761bfbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435808.92/warc/39703318cea2ec04f4d00fdd1d411399878c442e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435811.28/warc/bc013f957a64de5f549035d74f97e1a88164c8c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435842.8/warc/bf925b412a4b0ae9ca573cd3b52e1e5f54339290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776435941.77/warc/24e046f20cc8fec3ace73d2b25cd71d143ee6733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776436274.65/warc/a11552e574c60de55b88dbce6d4c498eef925f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776437232.85/warc/0b38ce427d58397cdb65ff3122f8618d9c9c8f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776437410.51/warc/4722b1e9a27ddd2be7f706f3055974951437ea43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776437493.50/warc/0c031f36e51656bc2b82c8283902882cef366fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776437601.53/warc/8169e845b81c9159c2d19ebafd086c599a0937be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776437611.34/warc/8ffef62ec7e9147512a08abbd198f0cd4f20cf64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438008.40/warc/5c6d6a857726f6232014b95fcf4bc995a221e7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438278.73/warc/2eee814c3838eea2c98924c5fe3237d13a7e2a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438296.25/warc/b456e27b6bb18aa9e728684e8580e5dd6c3d42e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438333.54/warc/1edd4433977a8d7fdecc6cab0a6e9f9f839655ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438382.45/warc/93e0b4de5fe0837c9e115726ea95d608a0068137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438441.49/warc/fb3ccfa8ec9d9b1ae773c1a0931cd643104e9cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438539.21/warc/31293f72c94495f2cf29d688f387a9e7459134a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438683.27/warc/1adff1e9f2ebfa475803a386ac7f9ac0e3e6ed6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776438940.80/warc/7599b38d66772bb2b3108364d2a5c56614ccfaeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439014.63/warc/71497503904b840fd5885688a325aa2885b0136f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439293.30/warc/1ea20e140f75dedaca25a61db8d5b17ea71021d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439565.18/warc/d3da87085fb256dfcd79f07213f8cd83dbf03383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439852.43/warc/e8ff27e295a460204704b5ef43c4f78e6708042c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439916.87/warc/eb8777af338d4907edd43b2b84e152facdf147fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776439950.90/warc/16342d173e585784cef5e3f186b0dacdf9c16868.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440024.38/warc/b1e5dad24873dbc2b25f9df26c123b34eed460f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440026.93/warc/b973493b5ee31b98b21d71eb2c9feb7d738e9131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440175.84/warc/aed1cb73320e29b7089a59ef90ec7ae36c3d9f94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440207.64/warc/5aa0d370eebebf7b67a22ea20fe3bfed68a7f1cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440271.67/warc/74996bc8009da27f19b2e11bb5c2a00c65272bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440364.17/warc/a4a61c7bb3e1a7d1a7aedcec5e1670d9d46a4a73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440593.58/warc/4a35bc839d184fe801effc2b4a214f108b817237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776440762.97/warc/161c825570ef86218b94dd1ef1079251b1e46263.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776441023.78/warc/3756bba2007fe362d68324cdb00b4302dfc1ff52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1404776442757.56/warc/2a3315817b7e4a7cdb84050eda254ac7d0dd489c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997857710.17/warc/2804550f0b762eb3388eda1d68fd3a1aed21eb3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997857710.17/warc/62e9fb73860938728c35be1bac6a8fee70632e48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997857714.64/warc/9c950202a865e0c7d6754ad3ad8212f06ca7e4f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997857714.64/warc/ed16e3867113ccff3346a67d6fa527139b7c1fde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997858581.26/warc/353876f42a23ff14f177112b469fd9f0600b4ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997858892.28/warc/37451e004f139f86bcb411ecd6915454488a8ae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997858892.28/warc/f92e44294a1e366425e2681bc935383a072e38c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997858962.69/warc/4ab050f42f8b2eb188c4a60f3d23feec58317b8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997859240.8/warc/36cac32a60bd71b18ec17305304a4ab44bde2d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997859240.8/warc/646729f9ef8da88b2d7cd3e350dc2a969df779a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997860453.15/warc/22f97ef29ad0f06fc97484b74ffba1043b800692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997862121.60/warc/3e617318b07a4d133248e0372cd20d391389adaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997862121.60/warc/67205efde8194512b4199ab35cab186d8bede4a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997862553.92/warc/8eeda543b19a634a60f6a9512f8e36b272a7b897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997862711.66/warc/5d2f23813371fbd3981dcc7b22f27d2afacef496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997862711.66/warc/cfd68922a38968a3260c2ac9dc8fae35bb387e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997865523.12/warc/1224f34e3899aae3134fe2c71c2c7e617e613aaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997869720.10/warc/6ea48a59d1e27598064d6ebbfbbb308d82aca5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997869720.10/warc/d342163f7fb457b30e8f77bc3adb886a6be886da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997869778.45/warc/2fe13d0a638906d65e7f90861329fc8c748bc3a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997869884.75/warc/22868c6ed823b209f9c013a855b3bd8326c5ebc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997869884.75/warc/d9af4d4e24ea057db1546adaf0c9917de33a7470.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997872002.22/warc/391c7f7bd56019ad51adc00e0040d990fa5526f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997872261.34/warc/22e03c13f1423f552d6532a0503f0e3c1a226123.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997872261.34/warc/ea735ce748efef915b830f25e8ae54c4cc8f85bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997873839.53/warc/2f67379293cc8302e89f82a7a8bd69f132d0825b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997874283.19/warc/3f93356fe0ffecb9f0e692bcbcdff7fe184e96e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997874283.19/warc/ba09147b7566241568e9d0022245962b490ebaa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997876165.43/warc/9de1c37975063ecb8b9011facbff9f8fcda4938a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877306.87/warc/59849dc7faf5c2479a4d78ca9434b3a182529bfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877306.87/warc/b55d7d2ea5663d382efe78c49b100297173673cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877644.62/warc/5c5dfaa194c8888096dbb3a79f398bd837eec0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877670.44/warc/345bccd98eff9e1c8bd92e723d46aaf0bb55ed2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877670.44/warc/63b91c3c86af6f3f43572f349126fac8984fa4f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877693.48/warc/dd75acc6f95dbe7d3a6ba1dca01cd125f91bbb4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877869.73/warc/1ae71c21b971838668487f6614f5670f44935644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877869.73/warc/801666f81be13e2a1da808b3d9379c1f572a7ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877881.80/warc/95503a57c2034232d6c9aad6a34a2657de884118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997877881.80/warc/f5b5eaa1cb0198b55170c93600a82033f4557171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997878430.52/warc/adcb184006b70f0d6eef6287dee5c15c6478cbc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997878430.52/warc/e59d3fb77fc56addedfe37caca440dceab247b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997878518.58/warc/7b0fc0c82d3a3af37f56aa0de79499c939a18c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997878518.58/warc/a887f5f9b548fecbebc22c0d88650402aef6eef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997879037.61/warc/bd1860f7c6859d33e781f357e9f14089b7df7d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997880800.37/warc/1e9aab6a65f60002bc3a52f8d7cdf75f51b470d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997880800.37/warc/901438b86d77879bf65fb82168a1260d6aa9da15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997881423.31/warc/917a92b3dd51c300e41c626ef983089258513afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997882928.29/warc/9b60bd2a8d2202d27d9e8d06a3d333cef81f4609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997882928.29/warc/9c43ada7f8475f7b695dcf8591ac1c855b3db36d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883425.83/warc/90ba40b9b35af5af1cd4e90e1592c940e7d61acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883466.67/warc/6ac28fb5e1372243f1e406ed469ae188bffd8861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883466.67/warc/d64637b6e92f4c5f8952bbcf1278dbc36ddb2b71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883468.51/warc/c9f9b4a1e52c6b25b860eaff2181e6652de46ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883858.16/warc/4a8ffb05d9e3a2a6f64cd42d5c19bef26a1efda0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883858.16/warc/ad77600a2cf770eb320ffc402c40e0927046e396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883898.33/warc/2b9294182b2e473955d84825d1d504b7d6a220ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883905.99/warc/07612a05b37b2f958e595188d3950ce399b36dcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997883905.99/warc/b86578baa3932c19b554a677aa328106aac84660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997884573.18/warc/50c3fdc6278debc4b337df74d1ec5d581954e6a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997884795.6/warc/b50e218d999de4bb629b7f72869b70b707d53fbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997884795.6/warc/bed3a5038c942feb22f197496da432d403362f1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997884827.82/warc/ed5438142c229297c64c47f1ccc23b1318e576bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997885796.93/warc/6958423c2cb82f997e10a0752988bbdb5bf2f8ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997885796.93/warc/f34f8473aad7e11d92745626ec7bd5830687b262.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997886087.7/warc/46be1cb6236ff82fa5f1f794b02913f17066177e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997887829.56/warc/2376ef8b25382c6208b0e460cfd2e8e305db1ff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997887829.56/warc/dc74905eb0488fa2ee4b5551d028b798190a0735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888210.96/warc/97ec479a5a67df6268dea348312e95d6aef81226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888216.78/warc/3898bff88c8d5edee95449a48a907983a3686dba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888216.78/warc/731584de81a2fa4f306b65ee22436d8d9fb44358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888236.74/warc/d80682453a3e50d464e6439e9b725d07957be2ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888283.14/warc/9603a8f99418117a7303692b21333bad985dfeb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888283.14/warc/b33896f51db875cbb5437930f7b80ec3194f79da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888303.30/warc/3132f9e377cbd9dbb096b6336b1eced85134d33d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888866.9/warc/60c1565a390f6273b74b62a619ca7667068d09a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888866.9/warc/af09e21f56b4add4d957b3eb77bf3a10fb10868f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888972.38/warc/3fd581a5820d268b32f641dcd8e2fa59591865a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997888972.38/warc/97eb2a912baf0ca5c56ae9c97790e376c973245a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889001.72/warc/2717cbc78344007c9afdfd8614341f1b038cc1a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889001.72/warc/689acfcb880d15d6d617db98860a50d24be4b5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889106.77/warc/5b5f3333f418fd9b026062b43f1a2b94c927fdb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889106.77/warc/77cc47e492bbe59c3c7237d45191b9d62102ed98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889255.54/warc/29bbba640aeade9aab841997f1514022d68bd85a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889314.41/warc/055e4180bfb72188d47bce25c76d15fc9291763d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889314.41/warc/9c826fe119ee32eed7bb40310bf8a2f1b5089b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889379.27/warc/db8102a919b301632116e29283f3cd1ba40872da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889455.65/warc/e0fd68c5d93fc2277e475ed2714310a5e7ea3986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889455.65/warc/f1b5e3a43cec42b7a112939fa564767cab5804a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997889622.85/warc/1c968a6cb4a5bf6fac7ebe650c65df0a9acf0f8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890181.84/warc/3795d87cf0a4ea0380330fc3a59a6dc4d85c12ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890181.84/warc/b1d0227c5ecafeece1b4628ea6320b9a4ffd40e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890199.39/warc/72482e4d478c8fd59a4433535f43e97b0687aaa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890211.21/warc/c030affaf09c5eb6df10a01dbadb305901e93594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890211.21/warc/fa0e425e51698ba5e5fdd3efbf789032b684adeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997890773.86/warc/5eb3ca7fb277c6aae4678af5f848b6bf445efb68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997891176.68/warc/0326731cb01f41796a7230ce6f43bff00f698f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997891176.68/warc/f2a879b49c1bc330b33ec8e20e8424863c1dbe9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997891953.98/warc/e4f08d070794764c053918d7ab3a4de544395f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892084.13/warc/461fa4ef3ea2d71e041543ab19629b28e9f5ab9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892084.13/warc/cab271f7563d10f71204bfbe7f3e9841b1ed35a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892495.1/warc/bec5befb8278d11c7dc790cfde6f73480138546e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892557.70/warc/254f7fe77af5fa5e05dc36566633cbb2e37c05c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892557.70/warc/25c918c5b8c15517a712d04c69a40c7a2d9215db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892641.2/warc/6d80d751dfbec8646f514410b5cf947a213e4fc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892648.47/warc/bdaab8cda06797e379923a568ae1fcb4f4a2f40f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892648.47/warc/c65b8e7a8bc745f5bd3b38fd7561cd288bd108b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997892806.35/warc/42f232e7d1a33df696b1b160248611746842d9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997893859.88/warc/b1e3fb0bf0663f492c1f71f5d2cfeed1bb893b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997893859.88/warc/f07a8cc92771cb743c69799d2eac7cd53316374d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997893881.91/warc/105a84e8bd2b68bf90f19f7c60d026745c0869bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997893907.95/warc/0e4e3529cb7533ea1996de4bfb8594a6b0b5ccb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997893907.95/warc/fc0f6a7ba7e39d37d47cd635705691c073add86c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894140.11/warc/9da681524e8318a723fe637da9656b870e43a5e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894151.32/warc/02d32be77fd6ded6879e8e7948906c6bfa35fc37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894151.32/warc/cd44b422e7e9ac6aca24f0e7ce83b9faf3d57854.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894250.40/warc/3514810992fa64fbdbe7e901136f14066465600b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894250.40/warc/f547f89c1df11af73f08825adbf643c521cbf252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894260.71/warc/72a5a21ccfccc9b030479a496b018e556d99c0c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894260.71/warc/9fd137f20b9dd38d42d62d0f8b62497bf5e0929f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894275.63/warc/27c7ff83c2334394e1a397dd4afbfb102d6e5bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894275.63/warc/4ec7a5de855a681357e83afaccbd78b9070169cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894289.49/warc/d5aafb9d3b53bee62250af3da9e4bc8011e12a9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894319.36/warc/456ab53b545cae350ac4105ac8ea8c7c0989bd8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894319.36/warc/785ece62305bda744d6b1be2c0c778606f409097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894378.97/warc/b411b4d1c73020daf679873754b83c03b7276a81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894473.81/warc/914151d1469c421f7b44ad9f1b09368de362ec34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894473.81/warc/ab4ca4c934b957f7ebda9cc5718dfd5c2d65cb94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894689.94/warc/9ec915d5768d7aa1a696071a3f37a6337ab0dc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894782.90/warc/2b529da7114dfc73ca462605aec1c58df8d102da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894782.90/warc/7e508545c4c4fe2d822f2baf82e639beb42bbeab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894799.55/warc/702817c776dc70910049640d92f8e7ecaf7cbcd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894865.50/warc/6baa335ce62eeda39d2217079c9705c19e2ba6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894865.50/warc/e2a50ad653f68f21e7f615dc2d223c1f3b74938d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894931.59/warc/da6d5349a7404e75dbbdf34360021ce23200dfd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894976.0/warc/8864dda1012e222f4f5876080b56cce8166cadd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894976.0/warc/f6b47253753120b045569722da3516e6629d054c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997894983.24/warc/d28b2d153e4502312935f623edd9acd18332b207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997895170.20/warc/177dcfd03d8cf761a5778f6c54ef778e161fd031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997895170.20/warc/fb83d6f7aa4189d3dda9bab542b41c8c75a9cfa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997900573.25/warc/6c623fe8aac406f2b85a29a19ab7a2453ab3501d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997901076.42/warc/0157a8c82902e8d616bf9f6bbffd895dc0fd67f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997901076.42/warc/7fa4dde988210c6faff4d6b568d703a533844a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997901589.57/warc/ddda94ef605a985899fda98ee418c5d907b5c56b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997902579.5/warc/426c8c402253420048fc212dd5519aadb301346e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997902579.5/warc/5de0b9b7a26b3b4b5cc96e4a4947e78cc06e209f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997903265.4/warc/6e14bf486fe75b5015ce3ca8e6a218127981d416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997904391.23/warc/4a371d97b86ac67cbf2f1322c512bda82803a983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1405997904391.23/warc/6bfd95dc4cf2adf3de0a46d481b4efe94eee3770.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256737.1/warc/5a46b830127b97bfc1b4257a52898d7f3c8dc3e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256737.1/warc/69fa064de11e85d751152ce1322e899734796086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256737.1/warc/718e0eafbdead2a89dfdce3f2a938bc75dc55e41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256737.1/warc/cb91f73c836e94cba9090e99b7bc0171c85ea84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256757.9/warc/3a78607dacef2db4ffe5378586c1ba5941e9e40b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256757.9/warc/5a62b2f281ee2f3bca22e75911773c84d11863c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256757.9/warc/a3006bf380699daddc0fa589b398e01447e9c15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510256757.9/warc/ccfb494276890eb4d87dd473ab78ab36134412a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510257966.18/warc/0af8b3c8450499a0e119353b67b357f1fc8ae54a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510257966.18/warc/320e1caae0d9ea426ca5dfb6b48ffd64990c93cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510257966.18/warc/3ccb006fd6141a6d554b79700454994c019be322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510257966.18/warc/c490fc6435419097bf0fc8035e0debafedff64f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510258086.28/warc/1cb05789da88b00a9c2c393a911a4fa2f73689a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510258086.28/warc/2dbfc6fabebd69b15dc9d728d56009ea300b5633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510258086.28/warc/64c032d0814b0470fbe5163090e73f062a0ad379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510258086.28/warc/876be4083b6b8f91c7d861715c28bf5532a2c926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510259834.25/warc/691d1f8a92c345be96dfa91d3baeaceff6d04b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510259834.25/warc/a6e5b86ffa7397f545eb0017a304bc4ee4944598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510259834.25/warc/c7aae23e54b631726f7cdfdfc62f9b614ba7e53c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510259834.25/warc/ed2c15e298182a6ae8b5707be3c2e48ea1856fb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510260734.19/warc/25a476197268e91a03475533c1d75ea63b631392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510260734.19/warc/4567ce16fdc61af50160ca7a0aa9ef446c109bf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510260734.19/warc/47a461d0b24c8806d08ef9ad6eb21b78df689b46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510260734.19/warc/63ee3a99910234d6ac3937c9bc78726c99fadfac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261249.37/warc/be73268ba192419295e9c8fa84e25cd06e7ede5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261249.37/warc/c0561a64dc64cbd70d3b2c82c75c32ce9a60f1d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261249.37/warc/ca8e90246067d9dba94b6c9bab05e535decf5ceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261249.37/warc/f61ecfb157365cd305be2900e6db93315d8e85dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261771.50/warc/705fd4ceb1d53c3f57889a93a51dcc1f2be18379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261771.50/warc/91e6f6584e3a2bd7198e54febb61e1b3c6e01fb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261771.50/warc/c159239d96ce3eed4548c89b0ab69b33399d36b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261771.50/warc/f56cda61c3523572d853a8d0f4a6cc0633d690a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261958.8/warc/17b02935d2d89892e33991ac1307616faedca592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261958.8/warc/55be1901cb6c1937451bbb594bcd47fdadce3f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261958.8/warc/67910e2123e1c81698a6a56cc7170e8d5527b27a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510261958.8/warc/ac7ce22c119c203d2288f3654dc25810c5e0b1ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510263423.17/warc/3e0fcdc85192c7bcec00ac312d22cb1f582a33e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510263423.17/warc/8d302bb249adf7e843f2ff06be3a9f0eeb2f1508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510263423.17/warc/c48983df42976134b5c763c78d8f35c5230d158b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510263423.17/warc/e5ed9d7b75114cac43585a3e71d3b082abbc49c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264270.11/warc/6131f83585d3473e9b19731c0a4967f52f644af1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264270.11/warc/b7714c3bfe3a4acc261f4ae13d8958f4c1d4c0fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264270.11/warc/dbcb412ed564f7e06896fa152e437d6b2f05461e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264270.11/warc/ed506a2bcc74f7493b83c9ad0cb61c986a84a370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264575.30/warc/64514344d50156fa8d58c8bfd586bc840dbbf666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264575.30/warc/68f303429be57b46257a2b67f406943f068ad5d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264575.30/warc/88a4d59703ffa3f0adb57e9f6a8bc4fc0f2e99a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510264575.30/warc/dc960cbec9a4f18fdd02903230822956bcecabc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510265454.51/warc/30e8cff926f0bc8d15a3290f976736c7a874a813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510265454.51/warc/88d5e8d0468a89cd49b6496a32fdbf5b098b549b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510265454.51/warc/c45bc9e1454259cdb67a4324dba43d4d1a2ca5c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510265454.51/warc/cb18e38284ffac3790d9fed4bf8ffc441891080e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266597.23/warc/56f620901542c132f2a915b4e28799323e536cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266597.23/warc/61e9f6a61ad4f9007825e651837d7cf089ef4ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266597.23/warc/6cfeea97eadf5e7114cc48602f8859512a6d4eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266894.52/warc/2743b5a142e6e8c9925b7281c239c0a1a5345c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266894.52/warc/41a03275d11d74abb22165731cf94d22bc5c9327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266894.52/warc/4b380f008d29c50efb0666f10ac0a0d79031bc0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510266894.52/warc/810c1f5df66b619932183708ba16e07d5c96600f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267075.55/warc/09b5d3f419450bd13e4ee7f3f15d6c08b0812aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267075.55/warc/531f5cff3d8bd8e046fd79e1c0d4f75cfaf15e18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267075.55/warc/bd6f75ff994f71e158ee6d47994d0cff4ef1bfa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267075.55/warc/ebd2fad203e806d923bb8bf29c6e4cf000443197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267330.29/warc/5fa163fb5f96d4ce7d1cfbd374070fc9b44148b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267330.29/warc/955893169ba750b4b4926f763084d878180d7c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267330.29/warc/c60e6f216552e9a1a9875ca455be4b973e8b8019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267330.29/warc/da467df6b19af0c22a4c1082834b58038770dabf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267729.10/warc/22fd45d16e060a5923df9f16845fc780a9894ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267729.10/warc/574aae653d9587b861e9d088479388a3af53e6ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267729.10/warc/8b2ccc90c88879bb66ea0d674113493fd9dd7689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267729.10/warc/e05e4ed42c65e2a4e04df8cf459c06393915d39a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267745.6/warc/4749189e0bf06b174efd94d6c97a6c8582b4db56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267745.6/warc/4bfc4c9be9e106d494e0a67d7624a508fd818bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267745.6/warc/ed80470271109857bf9028ca4b0e0f695bbe1a5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267745.6/warc/f99cc22abf757e38a8f79a87119052396c598316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267824.47/warc/35e3021d390a3db46a8e944880da5bc28675cfa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267824.47/warc/56781e04da953638c4d4bb067e39f582f1adc3e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267824.47/warc/8f2aba177d34ba4d8add750604a1974ad1a635bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267824.47/warc/9e82961221a984f5cd131419bcbdd9f2fb7f2876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267865.20/warc/2dccef76df36fe5fe6646db5ecc6a7e09d9196c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267865.20/warc/5313e67d41b56cd3a7d73580580c6f11f1ea433b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267865.20/warc/e6099535d31c2ad067bf9db7e327bea45e0a84d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267865.20/warc/fe7d42c437046f88129f469342f4c440e30443ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267876.49/warc/27c5366d6557504ee1d6b3d8963697a2dcf805a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267876.49/warc/5597b1b2c315def3345008ca4545e71f30c626d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267876.49/warc/5d5bd1403d0055d54afe747bb65a05d302cdaa2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510267876.49/warc/95dbf7094167236f79ac775cded9ab03bd9c8ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268363.15/warc/6fa9fa8a15ea7c1e8d86f3efb7357960fb1132c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268363.15/warc/77855c90d0b7b2091c0d2469114d651bb4e8d48a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268363.15/warc/88b8ec5dc1a9ea4b46cb6bd2c877cac9884ee4cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268363.15/warc/ecb825ee8513431ae7144353ec7e404091d5eb89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268533.31/warc/241e9ba0ce95429c8080fb6606b329b4c1d1158a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268533.31/warc/4a8f80b74f4b594fc3e54dd5cb2ee8af05fe30ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268533.31/warc/c169eeb276c6fcc3a3292845e53ffdcc8299c4b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268533.31/warc/fb63713f95f183990d081275071ab7ccf1fe7625.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268660.14/warc/60c4db5d8d4b3a95f6c27d597a5f27a9361a465d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268660.14/warc/afc71cb777b7b99790f4c84157701cb51921ebe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268660.14/warc/b8ba264f4237f7a1c67f9055a1302bf42c920fe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268660.14/warc/f364fdfdf00cc9b38584814adfa1bc1dba252aed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268734.38/warc/297b25f02bb2987bd2f7091ccbfcfffa9588915d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268734.38/warc/3fe5099dd0cac427063e6f78514cfd8534559f94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268734.38/warc/971b99c1e3aaf7daaa20f2cd00d5197a07fb9ed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510268734.38/warc/c98721acc5511a5cfb68fc7982e63ea6623c0cd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270313.12/warc/8d0cc51c559840e4db9ae2d73cce710cab95bd22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270313.12/warc/d42bb635527130b6807dfbe651e634dc0b160deb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270313.12/warc/eb5a7a73604277587e5a5de10928280162bef011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270313.12/warc/fc8094f2c5cd65e52697ec0972846617c2614b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270399.7/warc/0ff2ff4f324c826f098ca4db16277423e3f46b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270399.7/warc/101b1feb140dc38536e3a3aa2eb5645874311c9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270399.7/warc/2dbb731a0084dde9b57d869104eca0ab6ea985e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270399.7/warc/b5d7fdeb91dfd350a0f99f3467a2af6685bd6053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270528.34/warc/02724977313ac39fc96e3f32d9f6baad47775463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270528.34/warc/68b5f796da08eeb1392f97fffdd4ae08c3390757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270528.34/warc/f87441be53b995db186b9c5674938ce5405d0044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270528.34/warc/faf4bd8d79671b78c476d9f1f61591d16e2dc3a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270577.0/warc/0c72c3d4c4d0becd836ec0e91ee71149a0efdc75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270577.0/warc/123c72cbaeee03d4a835a1279824ed058d7cb9d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270577.0/warc/7ef298e68d94a93afd7bf28fc61235cff09add0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270577.0/warc/e104ba0bcc0f590e20c9dfd2e77e5b377f77d447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270877.35/warc/2bbfe2ceb014b52bff6a4c2e3de500416c3efb12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270877.35/warc/6a5570aeba72e35d573c459e4e217bdca549e20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270877.35/warc/d58478efecc4603a01c5109b01d82af9f3a710b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510270877.35/warc/f6a336005c05f281d5add753816a6c45919e6c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271648.4/warc/0a1ec3d4b7949e9fcad9017ac506e1229200166f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271648.4/warc/1839dd0360499b9892628f06b6ff8481a1429a59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271648.4/warc/4109e9d5e676cbddcdde5f4a36eacbbf8a09c6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271648.4/warc/74f3141242148a55f9c3ee60b6847815cf2c3fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271654.40/warc/1f6e6f2ebbb6b70fe9977a6e2b4e3ff32f0d1e21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271654.40/warc/9a364ac5ddd87faf416c95f43b5a3b350cbc0c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271654.40/warc/b56a8c79beeb1074bc62b86f208abd5e05bf1f4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271654.40/warc/cf22c9fa23150388ddfd82f61dc7a17685d73128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271862.24/warc/4e072981eb5f8e8a936968a2e862d69fadc47132.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271862.24/warc/5e609f6b964ca5083d5fdceefd656173aee956e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271862.24/warc/c0e053a3266192bbbbffd3c7018a902efa9536e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510271862.24/warc/ff21feb28e1c6522dd9e8544e0c03de0612cc3a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272256.16/warc/be035a4adacaa6ff3c4c44c3487543cda893bfa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272256.16/warc/dcb74132c596c9127c149dd62f965cd6ad609ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272256.16/warc/df18be2973250570f8b3008ad5922e580ab30930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272256.16/warc/e9827640e697366ed99289c0e2af6165589aa211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272329.26/warc/470545561661848263b521d36b9e7947f13933b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272329.26/warc/5143b62c175e93c49698b141bb8c24700b910eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272329.26/warc/5eb5ab8a3fdd6f469df9c859f70fdf4e9bbcc160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272329.26/warc/836ce4b6dc70e7172bef12cd869eac2c1cc29ea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272584.13/warc/129e7e61d40567e590b390f6c76481cb686f0d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272584.13/warc/2e3b78df01ca4343ee8798e37b3051474140849d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272584.13/warc/3e38041c82f597479e77716c350892db71e340cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272584.13/warc/e5abe3a1244a204a421c35d353a03a22cb9357f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272680.42/warc/0409448c4cf9b980f088cc73737cc275893e3ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272680.42/warc/51df2e6cb50d592ca1a84029494f3001feed74b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272680.42/warc/5c5ea95238531a1faee4f1fb7c5c4f61ab551bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272680.42/warc/6b08025e770f83286489d14ef26078b0c24de24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272940.33/warc/79c2ce190b08d4e6c8e05e5e7e9c5922e6a949ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272940.33/warc/ccdf2978c7426da9866a6153596375c1751234d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272940.33/warc/cd3c150955608d4c40e3468fed8d26752cf59141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510272940.33/warc/e06bc360b44047cb52350f794922d9779a87706a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273012.22/warc/29a6355df34caa52718c9b32e052de2dd3ffb8ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273012.22/warc/5b5f4ce30d42f1ef5a249d1624791523f39d74fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273012.22/warc/718ea462a9f70b33006df30387c1c51291a3038d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273012.22/warc/9ce965f6d0f172798ba043936f428f32c3a83899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273350.41/warc/0b31de798f115b4003f385348c2471d897c5d5bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273350.41/warc/37c779c0ce37811e2a524fb1fabd632cb0a8e1b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273350.41/warc/5adb1ae181d537ac077faeb1e91b9917ee14786d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273350.41/warc/79c3fac84d3819ed185f450b0c888aeb5e543613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273381.44/warc/23d0009d4565734c356ce7b5080c27168ab914b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273381.44/warc/a611f5ddb2d6cdf059e16a3b50faae3167ea9365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273381.44/warc/ca9d871a3d69fad2e7f9a752e7d369c7803d3fdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273381.44/warc/ff2e14eec59273c6912dc878aa7476b905f94eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273513.48/warc/633d59ce6bcaae869de402a0f96003be88cba816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273513.48/warc/bce53b5695635d16f63eca781ad7b3ba9da67b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273513.48/warc/de1995da771d6cc5e0febc5e26c386b68cd7d97b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273513.48/warc/ec439a1bda36094b69e707e0bb19d4209c48f5a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273663.2/warc/7e38dfcd18fd95e3bf5c8f3053d1e5c06a0ab9cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273663.2/warc/98722a1a0a3aeba990f3b4fb4f85bb0312da2fb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273663.2/warc/b23a85cfd2cef634b0c1188c82702f1627bd46eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273663.2/warc/c47e02f676b5720ef029e025b5319c2ca03a5bd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273676.54/warc/5ed338cc55af455bef4bbe5c5eae59d77b862f92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273676.54/warc/c23fd8fe890019da59cbc998fccd139d038d8d80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273676.54/warc/f0e8f78044cfce13c948fe1ba33fba951d186c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273676.54/warc/f4c28fd89ce118b0d7a657fd8c414b146be05b72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273766.32/warc/4f33dfba0daba8437abfe2dbf80753123bd66522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273766.32/warc/727e3770fc1720734e5f2baa0a7fc7c82db1a688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273766.32/warc/9fb81c125118a8ec90638a78f250eb64369dea06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273766.32/warc/a01c00145379af63f40be24ecd3d9a58cea29b08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273874.36/warc/684da086545ddd3313cc76c3c3aebcecda0c91ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273874.36/warc/8437fcc13e7eba465f51e53ed64f26d1010de488.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510273874.36/warc/ad0f1667330bb728c5b6453b28a37837d684bf7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274289.5/warc/4038a1c1c1e315fcaa2917ecf5a453acef3e08be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274289.5/warc/6bedfd87bcc694496868daf5863e22017b26360e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274289.5/warc/82bfeb44bb3079dde2350511dcaba70ebd549825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274289.5/warc/e57c3e2136b859e64ff0f8aafcbed3fe74de1294.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274581.53/warc/569169617a296e1da5cf989766cfc04bba55c844.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274581.53/warc/6503e4b3be45006c46b78b04024b7434ede83eda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274581.53/warc/6935e463420d036b2ca54cc3c3fd901072defb80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274581.53/warc/d12356dc52eac82a090777c47fb86dbcf9f47ff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274866.27/warc/3a30dd6a0f631316c5f55ba3c9d3cec3e92d38a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274866.27/warc/6f571fdcd170990cbfc3ed188ea337f56785babb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274866.27/warc/8d0359716044a5b7e1a9e5b2a2a70a7571e9a461.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274866.27/warc/8fb310ad01f355f96c058591a216c604258fdf92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274967.3/warc/0410e83bbcf66eb3ffa50a4d84a884697a83ffe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274967.3/warc/76f6c6208d87879ee99a9d1ee2d69dcd9704b308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274967.3/warc/ae6d2712e1538516950c872b154fc8e9d5c2b68c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274967.3/warc/c3bdecf5a82f53450e1e87ee71730665f156d4d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274979.56/warc/050b61a5df405c2681be19f00988afd5ed7050d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274979.56/warc/3cd3771a1ac98406bed1ac74e7161e729cbc5649.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274979.56/warc/51747ca121301178bd3e93ded44c2879dbe987d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274979.56/warc/9c734fe7c3261805670282e6925b083b9d3b3730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274987.43/warc/04f1e7e5675f560dad4faae90870e314590060d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274987.43/warc/0f67a24ccded071efad008369136871249229dfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274987.43/warc/2a0b6da6c34cafff5d5bd64bca4552234b836037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510274987.43/warc/bb93edcb93db2ff39ec1b4e5b19ba3e290076b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275111.45/warc/06ad36f8c654d5b2b615c5e69da3e52fe82d998e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275111.45/warc/1fcae5b9b258ea3df3d8d648ee8b444d07544b81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275111.45/warc/48c2df597f273f441d6aa6c331121c32178e5c64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275111.45/warc/962323c27e35828dfebbd773c032c92c60554ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275393.46/warc/445a2e17225de74be59f1d84bd28cb2a9609fa7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275393.46/warc/585af027f7bc886ae81a0fd6ab3dc9e8c2aa1820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275393.46/warc/7711702185a5aafcfe4405a6f61850f7f94a47ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275393.46/warc/ec6433f1fc13373687c222925a36e5cd1a342e1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275463.39/warc/68c9a7d1626fedabd51bb678c66ded0c4b482338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275463.39/warc/8e7a3ab5e9ced450120e51cde90bb720c0a9f3d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275463.39/warc/be74e6344d8ee3cbfc4f6f5c6c9fed2bbcbafa35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510275463.39/warc/e1a1a8426428831ecc309d3c9d45b5c110055faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276250.57/warc/3d589e9979a11a690806bff4323ac6d640c5ae27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276250.57/warc/790fae63df4afe386a0c2e3c7951d537a3033d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276250.57/warc/7a5711927de28aa075ab1a44499ae286ce05edf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276250.57/warc/bf7d05ae6e537812773bcf3eff90b20b2f2b957c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276353.59/warc/10468cfe6aa40262f0686c25cf98153286347bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276353.59/warc/67b03f96c59413650faea78d70170e7f5e7bb1d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276353.59/warc/b5945fa6bcf1cef2d322c7f780f6f32154e5da42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276353.59/warc/c00f51d695e91f2a89ce0bd34e16067ec3b85f6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276584.58/warc/05cd2655deb86ba52884ce31ede7f1ab48e9c039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276584.58/warc/3d3fc4e0c818ddf787eb4419bbd3bfc0b268ba80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276584.58/warc/66b32d4b25680055503feb66f0373654a871a584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510276584.58/warc/918fccfc8de819df9c5c91f84aca03211ae36955.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510280868.21/warc/22a2f2cd9222e04c4716147613a0c5de9d0dfabd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510280868.21/warc/80be330dd7686ac1e27ec234fd4ee1d968ec3f1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510280868.21/warc/936d112ec92506c53b5a37c585a9a6910dacf56d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510280868.21/warc/9bd6579bb24e09fcaf4c9f11f20c456d0cfc36f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1406510280868.21/warc/a40b9baa3636517cd6c3108c329daa10178174b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/14d228dd214aaaf0bf9e142ebcb64dab5d19186d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/14f4332c9aa1d3b1e8e79ee00b8c1b1c9750bdab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/16e3a4ea7cc89b5a0ff880f0855aac3430304c29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/172f491bf15c2ba0234d353fda5ebaf1effaebf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1a599ad5c4494be2e1a9f935aaa8423dc50ce070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/1fb4fbcf87a149061ee871e3ebc3dc2b10bc041e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2031f830da1ce87b99aee386e2cfd8b47665cea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2306d30ac7190eb52ba9e51c92fba03a20bc2cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2345366aa28592e03db04db7b44bc9be75965db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/240e892a8534a88a6f10ec9a8b205b523f3d6b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/275366d346f565cb68742d4a012682b0c6fdbe04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2aebd15f9d166e5cb9157ccf8ec30f7eb4c20684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2ba18b6a73dcaa4927dcacf75c5b946b3e49d299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/2eab680c6f881bb54cc4f890129eedaa35d03090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/30efba5c4bc044b54b14cecfbd8e52c42550b3b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3148a2a31e41c697b47c99bb6292677f06310094.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3161ef51418fc84ece373a8e6a469db54c9542da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/31d9c0eb80117a385fe7f7a0f9f9aa51f2a364fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3346f93c9995ca1b331528b1a36d099447069eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3376d9f93fe49cccdf5519f6652e487c9b87a18a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/36949eed52da9b3e922caf55559d4d6507a169a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/38682633b61f23012c5d9347c5cd89550550bfe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/38bb02a3ffc8cba6ff248d4114546c92d94eeba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/38be3516cd29106aabeab878aeff32e250507f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3bb6ea448c7e6c2b7024682c4bfb89ad2d0344ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3d0dfda182b551d3952e82f4427020656a033ca6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/3ee7451a1477f15604355413caa71fcca8c640b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/42d2d71ddadb585fa14a7d3c46d4f6468fe3eed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/44ac6876186a014a42c157b107eddfa397345126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/453f383ba90354ebd7a31eef7fa0595889e5464b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4664854fbfa978bb8517807396d5ff282f0c8067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/46db31a9d6ca31b66c87599370966a3b34ad7bdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/46f872c56e9855d6a9ef3e4dd5241de28eccce63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4860c1abcc382f01dce2aaab456ab73679b2e5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4a3abdd1caceee46289c24d4a5a8a655c623010f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4b742ebd335ba31fb389973055812763aa8dcc1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4ba41c5cf1a06a048c7c81ddf1e895c6e38b69c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/4eb308af6add6053609e8085e564346350973bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/52048cc8436d1e432778bc86242825e514d1aff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/553d861699afa707be9b66df9668c25ae023c03d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/559f6761fddd5834260e6d8014089e3fa68e9966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/58e20ea4a5f820b2dd1ba21452a4ceebf8a9df95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/594ccc04e6e47519e9a2595d70e58c4fbea8b082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/5eae666cb62c3ae662da5fcfce0259f5042244d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/5f4fe46442af734bde44603717c47c31b72902d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/61a482df0f7a034b9fdca02a31080e4b1473932a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/62eb7885bf25163c4de1c3b60f7bcbc46f891e88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/63e603c1fa583f46cbda1fc20c6102aec27e249f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/64700521a70e768828ec736f27a7db7fd060ea30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/688082d7bba89a517766289e82df7e6db8102460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/6a5a1914d4ee75c637c7660f5fe6b785ec54af6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/6cda89511fa6b67619454a28db861a5ca64719b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/6e93e16c8e6d83362669b3f8be30508852cd5910.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/6f2bbf24ac6d36947d18548fb1507b0165967f6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/6fb747b0277921a8163b7c0bcdd29317f66debfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/71fea35239da7da29510980c64562409e9ad2e0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/73413d85fa777137edcc080d666d123f6be89a43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/7614cbffa25558f19c2d4f87cda7cb9ff63a451c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/783a9d22658c580f93d0078293042ddbf1a26413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/7a4ff9103413fe4cec2372f49ae4798c9af57412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/7cf4b8a76cc53821403fa36c3f861630a1b3eaaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/7f881ff64ad43457b5300194df812474d11c97d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/80b50cde8fdb307e66e4d6c45dc3e0413faf9bb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/80f9183e750ee482dac5a5d14ef448be2c965e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/819bf264fbdd9e242162f245738cb4a9bcde10ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/838a78453ce9a2189fd4ccd29f4e42b368d0bceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/87eb6c95e7752122e4991741b43e7d406ea5b075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/894857fe8e997765ea6db77a210c4d03fd075f9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/8c8bc7e75f3b8db16a295b02ffca81dda51bbe17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/8d263f0e87ba1135360bd2438a6a56a9b2226006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/8d5755474808e1219fc64d62456683452797326b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/940d0b1b9b3ba16e9f1c9997ac6324136b744694.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/96b6c3875a918fbf18c3d9df7a851a0821cb185e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/991c54fe9181bc7c299bf6678c86ee5c94e4744d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/99d355d894a041268ae86d9f13f675aef2c730f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9ac820ecfb36ec74c303cb0d43e5b177460cdd85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9cbcc42550a85045627a339866768c598313e706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9d6b80204303b1d7f2bd00ee0dcbf3c0f76bbfbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9d71c405076813b80c7eab50d5787652515bbc8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9dfded6f15791339ef8fcb998a75bb86397a7b7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/9ed3f0df5f56132e4fc767ac5f123eaf8243c3c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a0757590acc322eb406e8865073749f5c95ce263.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a165948af24783cf806fe79feeb8ea559af9d867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a1acea4018334509e14e8fb5f96006870ce08b84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a45d21a187dac357c63f15ef924e87e98f290d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a4a513407f9cbcb592fde4d30644cf7dc38252c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a6bba85dc3323b748e241d551e439ff715eff33a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/a735fdc4c6b081716a3d4156f98380e22f65d4d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/ab6b4e1f951996e98258b36e1d01d167bb609f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/acb937856394362a8e03a401f3be214f222e2e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/ae2f4791b9dab13fd476e5c010f8d9b166239fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/ae662b6269591f96df5eb53a75e87416fd0c6c94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/af9af0951d1029ec4dd489c940490a6068e1567e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/afd758c4a60d9359830cbbc32a0971fc04772edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/aff8feb79f4e83f887d56a0527698a414948a696.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b0481ed4fbbae37f6afed8e66fd771e3909181df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b06d3b4b9188292cb3ee5a50311dc99878a5b99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b1437428feccc908288723cea520fed4fc91bfab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b22a8b35dcd85c100925d86c5c6a08b407771fa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b3e20908ce63200f96471450cf311a2319e1132a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b3f61050e93fdf751d6fb833f20d40e1c36e43a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b60b4ca94f1840c253f297801e3eee0ae6479d8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b6787597b49c3e492b82e4d2e067b1d2129ee275.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/b95d09f223a94e4e00aa4b9c29a7e21bc88649b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/baf748b4ad63e056c3c1d06e722dc30afdc202e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/bd27bcb964791e2845b996505162534aed420eb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c0257a649cd96455036f0c686ff9adb43a6ff14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c10a6b02eff2b984a044820dd6664494acba7db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c19d5d2b092a874d75510fc8709e1707b2d1f268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c1d0038c3da001f0d450d12ccb670e64ffb33d87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c32e8dbe209c9d898aa907a589a19d8c6006ec39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/c871bdf5d06f095d157a930e73c4333ac2cf5042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/ca38d60d96e039e8f79a63fb4d017a53f1fff11b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/cda43d985cbfb88a2d2a0d03aabbb668aba086e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/cef7344a61cf97a34fabfd08a4007574f6c1a926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/d05cd61b8a93ab477dc13aa5be6666ff5039d1e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/d3f8f9776721b3aaae55ab842765c673f5b85c15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/d964354fb8ef61604bfa918d15bac0473785dd23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/db9312009775b241cb66fa5d80d800e6f3c4a5e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/dbb0dc34f1029bd028d49791c12b6f3d4a37c716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/de7f58100371fc0e7b6db8a7af93cd9b43e0beb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/df7632c0768f6f721bc3fb37b078cf9c2b966f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e0d9cdc9e645f76731f5f4e29eeaa7da1a7e934a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e1b835abe5212ed3efba9186ffcda33e68ae70ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e221cb04831a95460e079f192cf746823b796937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e246380325642d9888e6eca00e0c8b12c9cfa99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e3b00c1b3cdbb9eb04a92b91fdcea9a3b1b146a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e5487e242cfe8748761ac1591a19a6308e21ed1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e594cedced4c4f6e7aa2ee29198d759a55d93aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e7d8ffbd7cbc703599489c5733b1677d46e56f85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/e80457442175437bfdb02affa98f5763c4da1c99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/ecd429ab35b828a70465754ee773d53c7559ab43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f1a250ba466338403a7f236eeba8ad8d162ed309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f2398dd439e60eda79eb8ccc5908b42c62517593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f3186e23a2909f230c1afcbd5b1ab984c160eab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f40a4c0f93afaf9bce2ff819d1600c9394c5a986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f6a1407dd595e296033c137dd8fbd7a6530f3991.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/f860ef545bf5a455ab665233bfebe92a35ccd4be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/fb3a2cf6aa7f9696a005283e7eb3bab3464aaac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/fd6878e9ee9b53623b9a1fe8731d568b61804ca1.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-23 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-23/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-23/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2014-35.yaml b/configs/cccc/mixer/CC-MAIN-2014-35.yaml new file mode 100644 index 00000000..71a13bcd --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2014-35.yaml @@ -0,0 +1,638 @@ + +streams: + - name: cccc-CC-MAIN-2014-35 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/005447e6d776d2d91129a84cdf3eaebc48b5ca17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/02daacefa71dbdb5a2f1311d250b411ca38d723b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/02f6b9ff5b2950fcc36a245ae2c5a859d1c8f6a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0937ed75f9a89fe7eb079d0df538c61d6a1dd6b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0b8935781de9d52511e6d21bfc33d77a6a8f7ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0b9b046d04011c69d604192ba52f07e43b1ee7e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0d316db22836180d5a62db74c27e080dbf8ae23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0db2fd943cbd45a95159260cd23a7d99c24e50f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/0e12f7dd88f208da931175b5ea12326551a15ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/119e247160213f7c62f48a8a1213155658da0350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/11ccff69ecef3d6a97d37acdd7216f91f28950a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1284801dce6ef89620c8ba357e65e3d6af564aff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/13ea67b9a70550f4e7a497558a29d86d640bfdca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800168.29/warc/1cb5e5a23b7a5f94a57e9c999851629799d2d933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800168.29/warc/760975be234c372580ef5aae2ddf977f4b95b22b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800168.29/warc/9fd5630f597f79c95c39df03b5d46ff2b092cabb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800168.29/warc/f6e9842e3c22fcf667184f4858f999f51fcd8067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800767.23/warc/15edf65bf4105d262187e8c49877edda92cc4249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800767.23/warc/65c24b0a3599276625c69144f4204304af675d50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800767.23/warc/8be61173483cec038cc298dd7bcf18ef99ef336e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500800767.23/warc/b19ef65884854c958dfe5376a36a20f07ff23bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500801235.4/warc/08228ee2d2e27e847e4860ff39d6ad482bab275d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500801235.4/warc/8195e4b529de9e4d5dbf0e2d0b4509396ee2d423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500801235.4/warc/97f9fbd48d2f31df8c121c63ce9ee003e0d9d3aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500801235.4/warc/e6cb4af3cae8b3ed5b4afaa2131da023fe78bc8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500804220.17/warc/b3464f1ab755061cacfedc92c7c92f5ea96cad2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500804220.17/warc/d26f5f63b222aab64af3d94d79bb6104e093e783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500804220.17/warc/ffe6f494878dd54589abb808d39ee5edc78b807c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500808153.1/warc/039ab38660cdf6549cac43c3eed3ac98179de26e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500808153.1/warc/15b99d3edf98880152a57e72174abaedecbbcfda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500808153.1/warc/71677360e334fec463447095b4c55d14a22e077d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500808153.1/warc/b3ad26e3fa18d6bd6660878e2f65a708a2e98dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500809686.31/warc/3ce08aa8af6892b99a1ba9bead013cdff83886de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500809686.31/warc/7ffc64a52a6526ff8a8e2c111bf3107ce776a2a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500809686.31/warc/9a6a19aef25d51cae0a95b24c83660089c927338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500809686.31/warc/eb39e2b2ca4763af642d9d4f5e62f019883f2072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811391.43/warc/216197ea3691669858fb53ec0618ca74ebe5495d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811391.43/warc/efc72c8a6565c1f587f1f4cdc99ffc4744ff4126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811391.43/warc/fcfa277deb764c01ce914adea0541e39a46bce50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811391.43/warc/ff46fd537a69be02c7e7f16a14943941db33055e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811913.46/warc/324409f2530ce383fbb8cf7662e026ba6e64ca5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811913.46/warc/b591ff53aa8e3824a0740f9f5a95cf06b6a01c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500811913.46/warc/e2bba5baa78cc2b378b9adf0bb894e0571dd1970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812662.69/warc/34f238944a33a55742ed8c2644a087441237f276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812662.69/warc/3ac6728560cdb270cdcf1b45e7998f0f055b94de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812662.69/warc/8373cc7266f7c9ae5fd7f5aa27933f2bd936507f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812662.69/warc/c66f55f8e6e04a6c3c291975e4157842bc71d444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812867.24/warc/010253e95eff9cda16d94fda0c174d2c248e3b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812867.24/warc/1f03d8d5c3c29f1c55e72d9324022ef95f4a8c94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812867.24/warc/4b455bb32e11e529bbb5a54e40e13153c09ddcd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500812867.24/warc/e96cbd58be7bef1b71a7fb591d2b1f2a80f8df11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813241.26/warc/02c447ef7a105d0008aab3d50b7264f3c8ee8e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813241.26/warc/42b067a6d33c2e4670bd94eb593de772f658439e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813241.26/warc/84d84e7063bf3eca24f5ab2c4df613ace2d76b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813241.26/warc/e3e6614fc7e83f84ad5ab2629d52328aec1f920c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813887.15/warc/1b29c606d941ec7f59d4044d793681a5a441afec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813887.15/warc/3bd897ad04e75db4c4022658192667513199e24a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500813887.15/warc/7f8aa66936fe3d6825bc9dcf7125ae6736a9b673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500814701.13/warc/6527a525a840d27e910a8bc6773ab721dc9515fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500814701.13/warc/75c4b0d827e8f6c3058c1fcc28586a187d132b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500814701.13/warc/7ac5aa597ebc6488e7690c471acacdbd2698d549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500814701.13/warc/fb90d4d69472e2298c9275b0fc75decffb2d3ed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815050.22/warc/1134cbd7eef11f35fc5d64a0f68debae027666b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815050.22/warc/472d6e2b91dbb13661a83c61c418c3ed905a654c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815050.22/warc/6b6406bea2f6dbc64ff27b2153920d2c425d426d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815050.22/warc/ab2106db3ccc01f1880bd9fb4585b23fd9698fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815756.79/warc/34ef3bcf1dab57473110926effced18ff30e80b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815756.79/warc/4c0886b351c1d3f634c8a902d290f1a893a93dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815756.79/warc/5764147b717eb1bfda854f7f38523060e721cead.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815756.79/warc/fcdffc9360adbc118c8c36e7927d6b1665fd18d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815861.64/warc/2244146d8276e542c173a778944083fb06f72f20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815861.64/warc/7caa610330e5b6bf2f75951704417c502a547d3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815861.64/warc/d4e3afd267bb988839401ab6d8e9daf5a5124d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815991.16/warc/160a2d9f8ca885350a5679c4ac4572ed17e5ee92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815991.16/warc/4b600ee1cfc6426559e3d8fd8d8a4c84090aacb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815991.16/warc/6a548f43f0673f2ef246b4acd36f035a74730195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500815991.16/warc/6afbac4979682fde3bd084379fede35a705e75a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500816424.18/warc/35397aeb15c81e46fc68224a1841a2f649ab0303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500816424.18/warc/618aca9b1b6f36afdbab1dc2130fe78bb9a30a41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500816424.18/warc/be7fda04725cf210259648983adf2184289022bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500816424.18/warc/f88af9502c8adc4a5a7865fd77c32be780a6dfb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820617.70/warc/63f4655a6648f044b74bfacf5797ce6bf9faef07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820617.70/warc/df976bcc24237d89f9336a6f572579062ac30ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820617.70/warc/ea2be38307925e29b82c027ce7d3e939682a50fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820617.70/warc/ee0b38b73e3f88e36654a1c1456bbe908a9147ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820886.32/warc/055e362a61d47261ee77eef86c5099d86002cad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820886.32/warc/b392913e6719864cfb1a32edbb5e39aa4230c1cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500820886.32/warc/e1dce94030774fa9e2780bec9b45520b56a74421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821289.49/warc/164efa540295a80fb5a2097d06d6a13d45dd010f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821289.49/warc/3c0633ecc5fadad92532bb534da4b47a244792c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821289.49/warc/c2e5cd09a8d0caa1897bb7d19d42ab2f8afefee0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821289.49/warc/fe48dc27c2b0588a7efb32a70288e0bee703d24a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821666.77/warc/0fe42a092589be773c5c4bfe54b1c007cdc610d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821666.77/warc/41ea4ad39ac329e944f0e66e303d2a06bb8a0ce0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821666.77/warc/460db4cd6d01ac5526017040aa99f45df66f50eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500821666.77/warc/d2a8384741f17ff15d7eaf066a09a89ea769b3be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822053.47/warc/22addb9a3381b9a2cb4578cba93d1367f51d3387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822053.47/warc/d2546e164dec1f7a2351ed9bc95824216d2956a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822053.47/warc/d9c201d959b22daf84958c15ca6423e7d76b883b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822053.47/warc/dba918be8e9bcf6cd7bf9e25acc017cd2780af90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822407.51/warc/a8b13086d6513d34c23147720351aaf3ac8c2e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822407.51/warc/bc0c1e58933eaa7367d7ec40f8616809de19fa26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822407.51/warc/cf976edb80f548b43c3a020871303fadc2f7f9cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822560.65/warc/635da12f966184a6aae2b2ee757a5ea8441f6de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822560.65/warc/7f4149202211e9e9c1825e0853337cb0cf707e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822560.65/warc/a07353443e3553c4335a087a6d615f2625372b88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500822560.65/warc/b9ef2550044334201629002a773c568dc608c3a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823169.67/warc/140facf157fa5d3b906f05fe3124ec4b76e21f8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823169.67/warc/3d879dc585836645138a662ca19233948dd0a7e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823169.67/warc/540dd82200cd6402f6a3a9c79b443ee35023350e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823169.67/warc/6fa84c7e0341fde495d6daa2fedb4f590ebfdc10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823333.10/warc/181d8b3cd5dfa8d8b9da2f4f6e8e68f96dd4ce6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823333.10/warc/52d76cabd0dd082a9a1e46ed9d5e63e9a77078d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823333.10/warc/a1bc77734c2ac81ef155a756c12052deb3bbf783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823528.84/warc/072f0e9954030d15795d559ace46afd633b3e75a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823528.84/warc/49686601b67f09c36bc7f038af649467307568eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823528.84/warc/7c6fe9d9656a710178ae8d6688cb060b922c8ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823528.84/warc/a174098a5a788c42e2f2b255691ef9056a6e0bb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823598.56/warc/0cbcfcd80b6e6534d4b0866ad6064d564dcfbaa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823598.56/warc/5873eedd2efdb7dee278ae77aa8c015fe95a5797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823598.56/warc/839cb209aedc4c9a3d56b1e339ef14cca5ce0e00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823598.56/warc/d9c4907157a211b8a94f1c1d7ecee95b499570e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823634.2/warc/51359162e8174a7c6652e0f9fd6d4fed1d689575.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823634.2/warc/56b663707c0d038d9506aeb375c7ccdb72492d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823634.2/warc/c108b009b4187df011cc321801acb8acab4379e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500823634.2/warc/da36577120ca96d0f98295748415afb0403cf47c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824209.82/warc/0a66c5a228564968510a73489f297d6a36f51377.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824209.82/warc/12cbcfc9cf8f334b875bbd726be21774c4dd0a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824209.82/warc/adaef14163524f6b4c59f53fa3559e77916609eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824391.7/warc/655fd83f645daef83c61ad5e25179fc79456b63b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824391.7/warc/6b8a1cb002c953b0c4b315186322b9e93228d5fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824391.7/warc/b0735ae0624301d93a56079270ab0bba34ae9c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824391.7/warc/ef127e7669f86007d77913c734843baf815e2059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824445.9/warc/0a1f4e64ee6fe593d4c4876ca5440eb4cfa6aa0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824445.9/warc/26aba7183a0830733538931901d2f86c02659a7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824445.9/warc/2c9fa6739e39b70c1109298dcaff56ed0f8b41bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824445.9/warc/a2a2dbeb56a85877cc5c27f21735f518f99be3b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824562.83/warc/263e4e06faaf944b2e293f472eef9dd5e08f3744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824562.83/warc/5ae361891babd247f1a2ff6cbee79b81ff44cf05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824562.83/warc/9045a3ac307e2c68832339ae68defab83d8e61a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824562.83/warc/d21b483767c9d2eab6762eb2dc7255ac1617f6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824970.20/warc/2f89b42e8453fa8a72e4820c38616ae78d62899d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824970.20/warc/31897813031831dc6f43a55a1db8e4651e4bf036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824970.20/warc/eb00399adfa9c447d2a17f17e8f2b87cb7b1abb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824990.54/warc/182076a9bb1cf25027b35418f105e05c3414adfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824990.54/warc/9087ffce398caa3f1e754f61ccd63c22fa28762b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824990.54/warc/b7c509d2c5b240f7e4ee3df3773c649449378e2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500824990.54/warc/fe8fab5338005281ca9087758171cb613dad6999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825010.41/warc/42530cf969612e0dfa28db39bdf151d82c142dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825010.41/warc/84f76abd7acfe92f806f1e0abb7d98d1a1ae085f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825010.41/warc/e49d3d67e028c35e863842f2d55bedd89d68d7ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825010.41/warc/e6f9d0036adfa9818d6117124138b30426164b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825174.42/warc/48750c44ef5739bf110f49b3c47c5ea960dab0d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825174.42/warc/78fd4bf54526aee75c6afa72a12e582ca77bd2b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825174.42/warc/d0fc46b233d1f5de35dfcaea83b133a5de28cee6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825174.42/warc/fe4fe31168f4618df7fbf15b373d9b37dffc81a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825341.30/warc/0477694b0d1d76281349b7dea290de8a2c97b2ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825341.30/warc/24393702e9bde24a74545fed64aca88ef919eb42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825341.30/warc/77558282f74cd12e36c897629334ec8d93593439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825567.38/warc/223198b7e293f7d9658d99d046fc523e1aaf2498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825567.38/warc/6ee954411b8d6fec0dbdd48456eeebbb662e69bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825567.38/warc/b276f7b43b802404c4d7b39fb9ea27a64376b98c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500825567.38/warc/efe4effbcf36b205e79db0b4436e677019bec42a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826016.5/warc/0834abc7c4f2cf89cebc33c0dd8a1c693da145ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826016.5/warc/1c78ddc32ef80f97c383f3829e6a37c0abd75ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826016.5/warc/482615ba877c1ee6d4a9e5d78928fde5395f0d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826016.5/warc/7776b84543e559f7242ddc396ea871db99d65dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826025.8/warc/22c5aad09eb04a7487f917d588d56768455779cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826025.8/warc/bb4745ef094d3aebfe9b1c8b231f294dd521cb9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826025.8/warc/d139e75224f138f1a0f63df30bb96ead13e79abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826025.8/warc/df07ad367aca1838c291f66f44e1ccfc9309f015.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826259.53/warc/061f5f665b652b269797a3f1c3c9950d1fec2ec4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826259.53/warc/15f17237f1528c7354fa07f7f2f0b7d0cba2c890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826259.53/warc/cce1cfe3bdf1c2414efbf0be36588d2afea997a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826322.0/warc/1f909dba0c57acdff8ad5f488b0ede6cb7a0c952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826322.0/warc/3949ebec8a9f47f3c1287ab45d3f0e468e3bd6e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826322.0/warc/79e5ba0a68bfd459a4db679c63b7b0e5b7d55b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826322.0/warc/cc638088e554a80c67b000cc569cb522ec8ccda1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826343.66/warc/2a80d8be3995e4f06c1eda1b3dbed34fbaa0fcc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826343.66/warc/4b7645c7e721ddc44c3c8e57ef3a3e085590e621.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826343.66/warc/4defce3668518e4fcfd24667b746219754e9e54f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826343.66/warc/c5638e374449c49ab7896fa4e1f03d4f53e39e8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826679.55/warc/1c95984871e0c86cd64d3baf0fe7ae646f5e77aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826679.55/warc/2a686c7aab06f9b8a9c6762249e0395367d6f2f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826679.55/warc/e3c155fa9420fa01e52c15da69bfebe332025c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500826679.55/warc/f272a3642a2ad6b072ba849be732d0b844717f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500828050.28/warc/2b07b5e07922c6f3d8b495e5d989e4271bc8d57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500828050.28/warc/e7a7ab2dd670a7a6e75e888028fa16213720c187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500828050.28/warc/feceabe94cd888fb47cc496af0625966ff68587b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829210.91/warc/14ffbbd704803f394d7b0eb0308fae8e7641a0c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829210.91/warc/44843c0fdc9bfe1fbd85911611cce7ce62396c25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829210.91/warc/643bc3934d7c7a28072a7f7c4258e81f33f8b15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829210.91/warc/8b6589d52a74424fda2827858c7a3123717e420f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829393.78/warc/197734cb19db721243e72f3e163b58b7dd25bdd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829393.78/warc/46c0fd90fa8aec1a4ba594619cf2ffa17e503852.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829393.78/warc/90ebc7e5a2193aa6a59ca6d21cf96ff8b56c7966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829393.78/warc/de4d71795686950f6c6583703db56e3a0d862b5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829421.59/warc/7e7a0a02a171272d156471759023452c10a5d205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829421.59/warc/cb314eab9a6acefc68b00d6ec990e40584981289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829421.59/warc/d4410d461833b761304fd3f2f9ebfd0cde19cc4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829421.59/warc/e168827c2c18dc8f00fe9b3d940839e466f2bf46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829661.96/warc/1f1b59f1811adf7738558f108db9a2ac879f8630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829661.96/warc/2cac7752432315230dec757733e293bdf25911fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829661.96/warc/982a60712b8f10a2917bfb26bffad2a85257fedf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829661.96/warc/bdcb4a657ef6cd3c771dfa34562b7af0d853af0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829754.11/warc/175983b0f648ed4754e27543525f361555c52a43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829754.11/warc/1fdb249a294b7459b02e1e84b26322f9fe48d850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829754.11/warc/27644b64a421b202e57dff7df39af5edc10a2460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829754.11/warc/37e32de02466a28ccfb3aa8a3389b8d0d1df9b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829839.93/warc/796b806232c43094b56d42b5e18b38c2a65d0b58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829839.93/warc/8d87214e1a564826f5068141d76ce0b4cdfdf2b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829839.93/warc/b6fe9ab9a57f14f4f288e986a91721e2d1a07f79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829839.93/warc/b8466c7a0617ff87a8856644897284d86b617c3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829916.85/warc/33defc04e201d6b70a7b0224404eb15305f8343d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829916.85/warc/591984bcca0332e300fb2f8ac4e19e3efcd50424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829916.85/warc/d27a5f49a180f83603bb62884cede9f5f9f0fa34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829955.75/warc/08e6757d21149f1dab625baa418ecdec8fc84800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829955.75/warc/1899da0cf7fa258b4a20b18a9e54475c5873c5c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829955.75/warc/5ae3c69b10797278bfd67efc85bc2d71ad81268d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500829955.75/warc/cb06a2a2d8e6e1b0cd17375eee5166899315749b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830074.72/warc/854acd820344755cf4868992af1d591c3f996e29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830074.72/warc/9d76f9f56b65684b4b5c1d31e258dcbe167e8e7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830074.72/warc/a829989825bf7aea464ee75f5b4431be2ecef8f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830074.72/warc/ea44c67209bf70ad9cbf323a43181aa8ca4656da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830094.68/warc/17147cc3c4d833ec21112272099bd361feb45505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830094.68/warc/441269396bd41b6e1e3be845654094180625cb0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830094.68/warc/82718f857cc5436930740310628827e2f6bd6075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830094.68/warc/ab2500d435031308edbb6a311d9eabb85de11d10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830323.35/warc/9a8f61729ad683df0025f422e30629d151ce565f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830323.35/warc/a26ae131c01b78b240a7bca16cacfad2f374227b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830323.35/warc/bf992a32e3fc95625c8b3327f41c0405716465c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830721.40/warc/0e997393de3c3771786788341ecddc642753c134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830721.40/warc/4894b8e84777afe82ca10f7e22c2a72c04e9d83e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830721.40/warc/a9c266492c068c2dc6426d3a10abb686209e6815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830721.40/warc/af77c86688cc735acfc9dadf5d7a59f89783d7fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830746.39/warc/658b07a67a827de7c0e22e66f53cd582c25ffdf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830746.39/warc/b850d8351036c71a1380a1ce5d873c38535263d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830746.39/warc/e37756cb6fb9d58cab30c755452650f27609cbc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830746.39/warc/f490fb9d44389aedb2b95f37103b5118d33b286f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830834.3/warc/2fbaf0e28a17d2df700e71317b5b23b1022bf19a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830834.3/warc/589482ff7f95654948059b7da11bc65385a63bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830834.3/warc/6aca7a8e4be2df75373ae06e381920761d2722f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830834.3/warc/800e6b6c3932cdf626de041ea64abb9430764cef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830903.34/warc/0819017139ebb6465932cd84afc61eb74439a0d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830903.34/warc/3452a6ce97ba0e667b2274095b7ed8986d2ad14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500830903.34/warc/47cc79f0f8ab62fc2bfbb3501c22334d3959ade2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831098.94/warc/2aeec726a874d53bc1c8210031d7155e22dc6ce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831098.94/warc/3212dd3901e080654fc4d534773fbbf20543470f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831098.94/warc/a9318916c34d26ef540fe9cc96c39db6e200e898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831098.94/warc/f06181a01bf0ff6c2ef97511b237e139384302f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831174.98/warc/2d77892299046d967cb6cafd7ab797247b868c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831174.98/warc/418918a219884a26f1459a385c22434928ce8de2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831174.98/warc/446de741ddbd3af1f2d6d22721de99fbd579c092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831174.98/warc/72e453a13ea9dc6604109d0002b2037d7398dcd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831565.57/warc/4da5eea2882a1ce67d1ffd86d798565519b392df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831565.57/warc/684013d94862670ea454be5c7723350f4ed903e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831565.57/warc/b89047260048eb4fd7d2b6b1be257772398cfd5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831565.57/warc/ff26ffda910281cd580a237b3861b1eed5879994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831903.50/warc/65b6f19d3227b17af0495880de471962c8c6eb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831903.50/warc/7a01b1a97b1f89e1f0bc30ae033a42b67c5b8e87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500831903.50/warc/a9fdeb1c2e0c9fa708b62f2e020807ad4830743e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832032.48/warc/1bf58e6960d2ff67e0f0fe88566a05e46d95c685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832032.48/warc/496a5f8ede53edb883719982d1ce5c5e7f40cbf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832032.48/warc/647c8f30d2be5d6e62e401391bb37d6683b80a36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832032.48/warc/86f73809df803bb9d6a2145dd5b6824295ed840e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832052.6/warc/795e448effb5d49d6659bfbe2e56747d26ff8c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832052.6/warc/7f622cede53055e13aff836cedcbb713923cc6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832052.6/warc/a4e38c1bb60cd82a85180a1ff05954e973d92357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832052.6/warc/c005c65e25fbd75249a6ddf0d9fd7b93e80a3557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832155.37/warc/3ebe8bc88690b56ad9ed657fbceb3b752e29d837.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832155.37/warc/aa22e0bd6d69800d6ed81479efdb8913277859f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832155.37/warc/c60dfba2c42709f1fe6adc28166b277bb8594d21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832155.37/warc/dce79e8df6fd5dbff3d4fb77b40c4ed586d2a8f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832538.99/warc/052488b50f4dc71039aaf2b72f062b447717cec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832538.99/warc/6cc62da3e83b39d1f204966951ff437412a2a1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832538.99/warc/d723eadf5c9d530f40bb9b1d6368ffc4c00acd18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832662.33/warc/236d04d9a14a329bbb1b32d39d7e79e10edb1e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832662.33/warc/362fd55a13d88ac1042ffc2a6a0c564b4576ecf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832662.33/warc/72b85e7dc3926cc267660c639578cf75e60d0209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832662.33/warc/bb3ec8ced20e8e4c62cec375971343add85ec760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832738.80/warc/176a66d62a169f918b72170587dc1736ae627268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832738.80/warc/37b7e6e2cdf6584531e189e373b23b884f57cb0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832738.80/warc/9c3f62006579fce88e2e443320d156ac915abd07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500832738.80/warc/e6d5340d27ecdb61cf7980ccd2195829fb42f2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833115.44/warc/4fdf6191af7955dcb929201e165adaaa565764a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833115.44/warc/5fa81117ebee0f0583832478692793b4c2bcc74f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833115.44/warc/9aa084618d209b649adae0ad3a479103b9efad73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833115.44/warc/ffc418d3c5bd151977f0abd3f6825ac89453532d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833461.95/warc/2b0e4ebe48a072ca93479544a907f8ec93ca2239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833461.95/warc/a8e95a896d9d66e24d77c6eab0ec5827ded5680f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833461.95/warc/b5b18c80db32f94444c8163902e31a70074e7afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833525.81/warc/484d377c544a02b4eb859b76bcbce929bc4538eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833525.81/warc/69d7759719aee58e7198ea590037cf7f7a373c94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833525.81/warc/f97a2770ed4bfdf6b226f3904c3c99bacebcea61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833525.81/warc/fe46cd41cbec0aa779849f280d33ae7101dcedb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833715.76/warc/2740081ae4efd8f32fbbf3ba7640fcd3bde177ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833715.76/warc/afbcae87e62534bc9af9fce60cb718054bbb7885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833715.76/warc/d47d06e14140796b602b323a3ef17411f8a4a725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500833715.76/warc/df144406f3ae96a181a42c36cf14fdf9f5a9845f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834258.45/warc/5ce213ef7b506c1259faea0b0aae47576095cd9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834258.45/warc/83b63c647637923de50283903ed2fe41b7600f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834258.45/warc/8db2a4b0259b3236130be2ed06141b127f5aa4d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834494.74/warc/0623e155ebbbd5a593b8d6a7ab34aad825b8af10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834494.74/warc/6b56bf7d9c44e1698a99b52a11f726ca850c5133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834494.74/warc/9554fa9e5db54446891f8d69fd92ebfd8d5e5e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834494.74/warc/fa598b47da4ddddc35017684f56054e40b2face2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834663.62/warc/1e3950110c57e4667425a9978fc6dc4cce789b66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834663.62/warc/3ee33068241b0359bad7ae1d97962a397ec483c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834663.62/warc/b40e9cb553e4c7c211ff2bbdb1afa7b6dd2e503d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834663.62/warc/c1395fd05c1fe34319c4025307005cc5499cc6a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834883.60/warc/0781b09fb375ec71dff08c0723f2344ecf8ad078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834883.60/warc/0eb30d88e661e249e73fa7f8a9c90071cce7f23b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834883.60/warc/d170c3c1fb79cd1ddc596706c5290b8747066f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500834883.60/warc/d6f1a8a7753fd21c70fdd80ca493ea1c02d97da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835119.25/warc/1700c75fe7e57e23a3150e71ae8c12964b3b62ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835119.25/warc/24ed4546153921b124913b42fdf6dd23d7a6b28b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835119.25/warc/b9744754b573f3486c631b5206e9ad27342feed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835488.52/warc/27fa59e45a5d13a88e822648f59c801624b0d5b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835488.52/warc/6275045abe65d41d13d7fa1b584772059a3d7935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835488.52/warc/d3e5ac3f4e7fa173da55ad8a12a0c06245fee287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835488.52/warc/f59343ffbf4f5e12589f95de9981b1247d1d7c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835505.87/warc/60004e582a874fe1cfda49d8b9ba901a69dc9c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835505.87/warc/601e024943ab54daabde39dd36b68dbcf33a7141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835505.87/warc/696563454d079ce898358cff00165b6aa468ab7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835505.87/warc/75db68ec80131e92fce83b682f86e5d2375a3019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835670.21/warc/40b3f7f6e89eda3e1bae811046bc38ba4e7a89a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835670.21/warc/5d55909aff9777e20c43595b1ceb49b5b1ed4489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835670.21/warc/8111aacb5a9818f5ff0e8dff09ed685c0803932c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835670.21/warc/cd16bb55efdb1026d14d01787ce37cbace0300f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835699.86/warc/5353dab9dbf6ba6b038cd4b6adb130816205ab05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835699.86/warc/6c684bb618dd5e957a08ae33dbe2e04ffed7e787.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835699.86/warc/bc5b80aa529fcec8324313ca7fee84c07a8facd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835822.36/warc/9102a8bb5802c644bd97ac564a89f9a1ce0a9392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835822.36/warc/92992daa2e4809227e7c29344f416460dd25aa8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835822.36/warc/c1c39307923af971b728b280c130b0768d0f6da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835822.36/warc/f2eab9dbe27fab3e69e31893de15435f5ebe8aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835844.19/warc/0dd968bf8bf0c25416ceee1068473d8adb47f8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835844.19/warc/6a393e9bf4878c68716e1f8ab82555bdce90478d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835844.19/warc/73da7a708a81fffea2e5466ea395885465fd0908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835844.19/warc/74f5f6e56c5497e2c6c96a6f29eb40fe436d57b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835872.63/warc/8d1846596b87e948fb07d59e4d3a8163918467a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835872.63/warc/c1d9e5296560190fde023267490bff62cc2d2b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835872.63/warc/ccc16b00961de7d8191ea1aa45f3a0f19b0cc549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500835872.63/warc/dc3de59a0ab829fe6b05659a79efc00095988a7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836106.97/warc/16bfe1accc91f8c3f7b90f103d787350ab2402a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836106.97/warc/a9d40567d36458efc5dc8838577fb5bfc0212553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836106.97/warc/cef6a1ff96d2815b7ea6d80bf96022f3b917836e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836108.12/warc/0409df0899b2a7313195f2df2a3a6555ac47ee9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836108.12/warc/15ba98b11fa7cdfff4808eebfba7bcf184e2123f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836108.12/warc/8eee9373003b22c7457db6e6ff53271008633f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500836108.12/warc/98e0bf85ddde3c1bf944c0c786e81254ca2b8f71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500837094.14/warc/43d997c3cc1b8e0b567da1003f2574ad5331ae00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500837094.14/warc/4cdc85f429e730a45d4cef2b560c3c9928de2de6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500837094.14/warc/a77d32257818061affac437a52b4b4b5253ffb9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500837094.14/warc/f41da9ed41f8b4e5480abd6f569e1b49f24b2ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500959239.73/warc/03436f67172b71971be8f9178be72dec22a34848.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500959239.73/warc/5d3b59b05788b6aae0726dfb5ea7fd35d00c2984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500959239.73/warc/72598c92a62ec44dd557cb0358b739c07d143ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1408500959239.73/warc/74c68885a9d62603b0a65bcc51b1cd7b3a383d57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917463.1/warc/35377153167332a55c20fe5efb8eca11967e34e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917463.1/warc/c18909e2237c13c06ff4703cc11009e87edc6c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917463.1/warc/c40d9ac98d6e3eb48916a353c94bb542a9b1f246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917663.12/warc/4d3cc3d0ab1f89a646999e013070280e50fdd364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917663.12/warc/bec1838920764a7c373a4e991833db204a10a07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917663.12/warc/dc9f9a0b3af6e2b10a5de296317193fa51330e2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535917663.12/warc/ed038c5b076bc2f4c47b311de593649be00ae69a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919066.8/warc/06ea2067a9128ac2b0320f485c8d98f2a7c542e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919066.8/warc/3ad5edc740ae950fc6d031e7c33da65b021fea02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919066.8/warc/75a47a5426d63b018a7f8800d8efc8c08a6fb8e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919066.8/warc/fe31880d3705f1263db6c18b57672fc1c6f9e380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919886.18/warc/2bae228ebf3982727eaca5f834961f651a587d3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919886.18/warc/4e81843c863a14521ee543ca6ddb8a6158429bda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919886.18/warc/ad582ff52b05a62270cb614a1f44324906bdb6ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535919886.18/warc/c05b85ac09e9f13a2a3b6c16978a7dd2da2536ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920694.0/warc/3d3d4f42f60102051e3a4f7d42479fc3df56ba46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920694.0/warc/92cb0f0bbbc1d0bebc0177b6e4c0b9bca6c5d077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920694.0/warc/ac49e621a1b73dd9437cbc3aa04407ff320dd76f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920694.0/warc/e38dafcb3c4730dc74ec4191d3d2b2aeecf3e386.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920849.16/warc/a9fe4ef310a96048691622e4aafaf7e282257106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920849.16/warc/c1661de9f07c5c447b0ba276ba775f575c1a1b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920849.16/warc/c5d1eda3861e35acdd25c934fd86f3ce44de0415.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535920849.16/warc/d75f84eeebfb57b96b7326b44aff4bc12d96b657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921318.10/warc/0043f9c6bee66d6d1549e1827af189c36d4d3653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921318.10/warc/08a8159645ea0c93a71ff381c9f49f334a78c571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921318.10/warc/eecd0059da919f01a50785da53b6ff25ff1db531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921318.10/warc/fab05b0d2b047a696a9a1be2e763f97ce2575ec6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921550.2/warc/40054c83f1611825a985af400d330d9d8e69eae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921550.2/warc/8e1d0acfe618a2bde0da572cd429fade5d24f016.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921550.2/warc/9f94e205cdedae827451e3dd4d9a6a813bb036e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921550.2/warc/dcae9407c7498706ef28732c6b2cf04b596255c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921869.7/warc/0ee98568d408dc26e969ea5baa41f701dd9349df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921869.7/warc/27841aab32575ab5a6718f0563fa5ab1dd4b44a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921869.7/warc/75ebbc178d9f54b08f94a51841fbd6c9056d02ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921872.11/warc/1e74d271fe884638544db8b575f80b3f760d4e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921872.11/warc/254fb2faf11044c9f85fffeeea45e9f746ecbe39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921872.11/warc/95d5ef5c44d47f1064f37646fb173f839185ebb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921872.11/warc/a18259086ccbdecfb3b2d160d2156a65749c4d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921957.9/warc/49003a302a602172a224dfd3a246c743e899c6eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921957.9/warc/8ef924bfd0b16ae491c1149ed778bb0858d1bf67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921957.9/warc/cfa74340f564c7fd4024df1472c9d48341f3a275.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535921957.9/warc/dcb196c2fb7ebf597eb4e1fddcfb43771ce928bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922087.15/warc/01b73d58f155b7cf642cd1526115b6003ac13d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922087.15/warc/14f8f91fbf68221162a76214416f90a15fce9f6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922087.15/warc/3e80d593d253a72f88084ba15dd131a95bd92881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922087.15/warc/56540c71bb3bd3f289b5d0ba3dbf9bb0c1fc6087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922089.6/warc/404de41186bcb25c7a45f7ac1e4dd9ef9aeaf434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922089.6/warc/4e5579b36bd9ac27fe5bf096b96484a34e40179f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922089.6/warc/a5b0ee6e734124f23d86c8537b4e0cd1c5a6bb56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922089.6/warc/b16d735a10c1a6d09fdd647311ba3c44d6052e9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922763.3/warc/45a5620522d97a05244c6205ed19a0a0b21f5c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922763.3/warc/b08cff7fdc4a4b8dc20833fae3e89d5171402d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922763.3/warc/cf370c6ca1de6ce137a755fbfbcd60d05bca6c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922763.3/warc/e30aaaa5d9496da19aca050aedb8dd7bba55a0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922871.14/warc/227533f7fb14456a840cd0964645fb1b186787f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922871.14/warc/45a945c41c90a656de919833f18678e999743a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922871.14/warc/8d2ce2069b9bed0b85f77e939bd2f31e52ffbd0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535922871.14/warc/c561512563683ffe512cc81a99b232344e6b140a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535923940.4/warc/2f2718b3d913f0bc591b3193ccb982fc4838528d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535923940.4/warc/46f4d60c4222edfab8b13c4fd2919aceb3a674c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535923940.4/warc/5f5a7c69a626ec86e85caa14b47fa778cb7b9c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535923940.4/warc/6a0b25faa066f2330766a5d76d9a4645b96bc7df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535923940.4/warc/a3f9a3fa2f0f06773186800eaa68d84d58661b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924131.19/warc/3f2faf29815bf6307a1eb6b23b887113f25f0b82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924131.19/warc/7d7dc8e669b8d3fa14d976b3e56e3ae23ca98008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924131.19/warc/88bb030c0f9180fc5aa4ec62eecb7fbf700747da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924131.19/warc/8bacf04f2094500a1bfa1f5135fe8db9ecd87ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924501.17/warc/4206689bd78e1ab17cfdf2fec08f184c5fbe6b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924501.17/warc/c044b12adca9aca13c755e099b783993d086a72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924501.17/warc/d856d2ad2817f609ebb8d2c52712d61cbf78975e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535924501.17/warc/f8ae70d68917e9326e0cf7c9a61a24290549da95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535925433.20/warc/422f297fa1c84b26eaecb6b2bbf19352b265fe29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535925433.20/warc/9aaa3c1dba90abfef7968b1d739fbd6f007584e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535925433.20/warc/ba92a9003692cc0732a27ac2cb1d8baeb528ad49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535925433.20/warc/bba045aabdd11278de991ff4a40c6bfe624a08ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1409535925433.20/warc/d3412e37459f6cc9ecc731167c95378b9ae24afe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/17f2413f43bbde0389803a7a7206c4727ceb2988.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1b99519eee322bab862293ec750daaacf504a5e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1ea9687f72d22bb43d79cd0a79230bddd3bf5afa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1f91f81e4cb8c0f4b32178b147e02e791b3bfd73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/1fc34287075cd830c9ee1be37ec1297232b74588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/293d995e5ee616cd5a86aa61cef13d7315b00b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/2a09d372d7ae50670244884952e3bc4f06f81a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/2a46bb48e992967e26c8bd46cf0e62888a395c76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/2c1b706d73d89f96f64a2e004c811a8cfc44bb39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/313470c0c45f510979912f55720e693e5c5d49be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/31aea39d8906291badb0ca2f9240900a1a7c3ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/32e2cf815fb5a1b22e725accd085b584aa375d1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/3740ef042f2f9ed104bbf231dd568308039fdbd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/3a56295a600f45e8426189452440ddd8c263b71d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/3ecd2f5c78ea26da0bd0c68b644ded8422dfa92f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/41cadea478dfed9018bdfd307334352b8203525e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/41da3ef19cd26be8689ad1145493c0cdde72b1e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/42cd1159b58dfbaa3bb8adf0d816092f14fbe008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/4c43b9a0cac78a41916e1bdedde343fcd148225d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/4d230dda3c4e8506115d76be3726444a4607a35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/4e1f1a66ac0a28907254433a47d3399899302f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/50230fe073a153b2e6f16074eab9f80bc2ae7a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/535f8c04927759cbc24e5f63c991a8631f509676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/53c2bb1b2f12de17a67c2b185d2e8ad0ccf55b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/540cb922aa24f7dc8044dbab6fa2edd1b491db51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/5b5d291bebb678fd2f4206355ca8abe1689dbc9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/63eca566219b104c09c679c3856207d44994f226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/68db0011e1edfb6e9e01c536b515510bf38efbfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/6a08cdb0cce4180d527e10b8560d02ed8f6de2cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/6a8ebf1d2244dadac77f0ae72c9a9294c6e033c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/6ad72ae65cb3961088a222763033d12f4de8562b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/6c09240d782c39321a959e8d4d6748efee29d93b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/6c3bc5faadad887782403fdd8e07844b4a08da64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/7048197bfa282fdd0c4fcd6cc9c0bfd914f2a531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/71eb412c9db4e0019522b26230722492be618804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/75cf562584c07df50ee9b127fce9621c8e499e86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/79175165028e7637c56273a051a479fc528af402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/8414e4b633cc52a3a403f46e19f80b91a8a07911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/851498e299ba7d4c0cf5c9b08397b3a73221b523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/851b15c719577d0b62bdc2788210e4f6ed4f1d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/86337872b7e94356157fdf2d3caa7c8adf149729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/87ffc4c13ed3cab6f4016e3ed59be511a49a44f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/8aca9cd6e2b62da4a4b17b59386501a674fb0044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/8d6c1ab7d5835266ba21ecdb0c62943c9e018d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/8db26b6631c683fd100267cef5024370fd7dfa0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/8fa4514775ed53e91b51f17d89d5b86fbe772b9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/9390087c31de056bd3d6c976de9a0cd74d157b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/97edddf28f7c8a23065f5ca8f55ba1225850f254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/9be8c325c7258e875e2fcad955ffe5c759de442c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/9c8e55c3c9b88b34c6d115a4e1572078fe97cec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/9fbd9ed596a9bd5ea878eed6629749f330fab3a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a092d62f2a0fbf1a80072e016952f4cc918dbdce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a0bd08961a935755f89b823cbdf6e6fae23519f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a2993df1668506f50ea80dc9846eac4b3cc8281e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a2dc68a5cfa0331226bdd10a576d08ea297be681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a572de1e26e03d90eaa22d0e93df124231a097ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a8413a4f58c1e9452389cf4179ff41d13d41dd0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/a9f074fada244fb31dcdd206e4e8f80da3356429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b04cc11c6238b65281f9267a426abc802b9e33df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b1af479d284c35bfd474e5f6baec31cbc30341ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b2249e2496c90062329211cf647dc18f303bde40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b55c49991f59e064d192aecb1ab33c6ba8cffcd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b6419d6cbf0150bded0f14c9314de6755cfe6b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b8b67e5f9e411228b5bc2d05b3f7ba0ca0e9af24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b9ced1c2f706350e90d7eca3694cc18400c6ac68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/b9de763d3c37d207814c99fb76fdd9848a11a734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ba2db8811c84d43b29c3ebf558623a9eb02e226b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ba6a7e5c53e30b971085fbd2fa602b0191e57e5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ba6c318cfacede4d07d7ecd3f4d50cc4a3fe5f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ba8853cef476f4129ed62095d2bf80a064926cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/bd0446e1242e79b8b54a5a2304cb952e172e031a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/bdf6cfb04fa90ca235a33cfac3073270644c7f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/be82b1a4a21b8505121571c35de7b2b83debbd3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/c72637927b2cce84004d2d48977e566d805e84c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/c8b0d930ecb9d68cc616a90c2f0daff72beea01f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/cb0f3add9ec3e43bd2bb763ecf1c02ae6e74a882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/cb88ed7e4ec9487f4bfd5c28ed142774422d65ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/d2356bec5bc638d0d95d6a330b6c2d61ade3c110.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/d2834affcbd5c3e74a2bb72a819108bf0032eda0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/d590bc076eb8ce6166c27ad85f7476c60467c338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/d5ba8ace7ea54e38073c04b5166277f990a7e05c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/de4a3249ec37007a9a29a30b96f92715c8264216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/df48bf6a2b8ecbce0d695d0fd7492dfc754497f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/dfca7b7a5f78fe03713fcd79129cc9de903bdfd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/e18d7e9bd08b578e81996bae9c68c3ea23aea701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/e5805fd2c46f73b4da31ee89d18d99bfd6702e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/e7182939322c283f38c38a1e22b8e0ac9eca8823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ea13ad2cb04d38836eb3bbb5ba59be10a924e0ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/eac47e301480ed1fdc387fe976234c9301a9f559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ed02ea81749625f20556cee23967f46b18daa96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/ef122c041e03d18555044b109dc64db973f4adcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/f04e55ab2f626da97cd7f32802de90bd7325b3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/fcc1b7dc2fdb5151531b20d22b09ea31ce7b6ef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/fcfe527294ff2ff59f7391c601dc93fc07d00ae6.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-35 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-35/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-35/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2014-41.yaml b/configs/cccc/mixer/CC-MAIN-2014-41.yaml new file mode 100644 index 00000000..fa413053 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2014-41.yaml @@ -0,0 +1,592 @@ + +streams: + - name: cccc-CC-MAIN-2014-41 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/0228c75513f37751ad3f0450c0479b642e993c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/0249a76ddb7bf44ff7b5496859ba93a545ae9389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/05a534973aec4e0691b4eea8b09732083b450b53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/08560c52d1b622a58a5a8e143e25129505811da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/097fc55dca09bdb47dbf18ae0070c1eec5e692e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/0baa92682d59a7a1f9bea258084ec50082511478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/0d22199ce5b52b04922599539254f3ffd3c692c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/0d420f7320da6b5df827912f4ef6fc85b25ee4f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1214cd1b7f6fbb7e48d7cc34c076e7c754852926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/13e0d878926899ae729515f7490602bbf11f61f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657102753.15/warc/670cb56945705898830ab5b3ecdfb4502f306fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657102753.15/warc/759541d6919c323183999c5d7a49f0bbbd546d5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657102753.15/warc/d61f7f9fbe76dfe8a6142403baddf8662ee85baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657104119.19/warc/23fee1f14b08483ff26532862494e77853ce346d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657104119.19/warc/c9f07874891ae182af38ae18333e82f28d79dd36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657104131.95/warc/17d182d8cf2c148b8965a33f4f24ed2341329c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657104131.95/warc/b7b55c352cf3288b111a9521d3a52d21f4451fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657104131.95/warc/f485f715cf69248f58ab98763923fb6d3344bd2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657110730.89/warc/45e42e34782ea4086b652142c95c679da1a802a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657110730.89/warc/8550c90cb04cb5f6c1ca0d787109808cbc4e050a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657111354.41/warc/019bb6c8f31f0212b08a56dee28126b2a60cc11c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657111354.41/warc/01e6c3d1e3184b1359d9a650a56efb3f7d187e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657111354.41/warc/8a5026857fe7bcff5b9e2e591bc4be14108353f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657113000.87/warc/23c9b22cc43fc91a5565608defca00c40da1d6e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657113000.87/warc/bdba7fede8731e759d947a940ee3ad46d9e019ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114105.77/warc/0a406cb768fd19879de48d9ed765666dd1a909eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114105.77/warc/6534e82085f505e0ac9bf74926050e5819bade3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114105.77/warc/a0d4f24815cafb910284473cc65fc440c5fdebff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114204.83/warc/bf0eb3db11400544e874895a752f4e62025963c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114204.83/warc/dadfb58ff549cc90d1f32d619cedb93be6a2f127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114926.36/warc/2b7634bf1b91c4d8cf90acf138aabef4accecd22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114926.36/warc/5f391eb27b7e362f50c2bfa515f7a02455d76133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657114926.36/warc/bebc57b16f8f0e4ea5c39cdb2ca65c6d112082e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657116650.48/warc/75d861297a18687abc711f126c243c323372d78e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657116650.48/warc/ec5f40b32249fab55761664a2d648c193dae7692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657118605.64/warc/0427cc23f1ba79c3a61e925d8a7b364ab010d6bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657118605.64/warc/075ad171b6916dd2f9a5de62412cf0eefe5a6e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657118605.64/warc/e894e1981c6e98a66d55a0dff96ae3bf06b48e64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657118950.27/warc/8424210af90df47caef6f270f18f6aac4b7e0d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657118950.27/warc/cc2e4d830e3d9c4d0b5cc2499a0565bd0809659f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657119220.53/warc/66b7601779d17adeaa09413cd3b50cb8525e8d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657119220.53/warc/739bd6150b85b00cdc0d29ed28695cd158b9a1f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657119220.53/warc/d1fe67462b89416ea8ed36ca9f41927e6bd9e5bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657119965.46/warc/277f00933226939d4b5c8c2cf1ab4134b8d170ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657119965.46/warc/60b00dba0874f3a57c1589eefbbd0e4cc37e906c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120057.96/warc/2ec4e46af5c795d1922e4852ce408734e9199c98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120057.96/warc/5d64370f190bd69016eb2a956e347492ec7ee3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120057.96/warc/cb14abc02ef964276d447c6224c7196567eb0fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120446.62/warc/82647b268dc2d73f5d5c6cd645093a4dd1a7e670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120446.62/warc/c3a9fdff328d983c11c48443a2e8fd1fdfef06fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120974.20/warc/659365359988c3a449b75b59d07fc7242e3056cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120974.20/warc/f464dfa4ad3ea4b217a9c94fdb7153275798e5c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657120974.20/warc/fc74080eb0ce7b950224550f4d364fb142dc66f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657121288.75/warc/34aff26123d2142ef816a5b879da896f198a0f97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657121288.75/warc/6f27e565346a68a281b1fc4c62f3b4c40a16b7f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657121798.11/warc/f2fbe73293f27f911030942798a35a59dae02085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657121798.11/warc/f3e7c7f70ffa0bce7bde671f9819545ee9bd0983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657121798.11/warc/f7a7b74efcba916a97af92e2d142940d48771501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123274.33/warc/0f1259cb779eb107283ffc2e348c5ce4cf6f6e0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123274.33/warc/edc3962fecc45f6b2f252953729e2655c6115d6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123284.86/warc/5c4833318ed3d24054ffb57d778a2799ce842f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123284.86/warc/f314090f3cbdba5285d9914ed33dd766539a0831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123284.86/warc/f616e5b4cffd33e312c2349b199ed5c54be54a8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123617.30/warc/4c5cf78e3768c36d07b68be6892c520d89da6251.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123617.30/warc/8ee7c7a8ab8271a2982d16a861cead9cb2feb174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123996.28/warc/6910616d664fa33d97dfcdb8608ccc0905ef9078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123996.28/warc/d2009c1fb09813e237fa5418c68683870c65da1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657123996.28/warc/e2def4afee58cb0b8817db2cd535f5cd892b33fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124236.90/warc/31ba841efa6f4e03aaa11701bf444f2b424625c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124236.90/warc/5f6916648653aa8fa56964d24d3406629015a82f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124356.76/warc/12f41aae5bd70912c9fa64c333849ce2fa30ab73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124356.76/warc/9f51162fecee54d0db38d61d524e77a55774a237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124356.76/warc/af615be9b77b93480d9c86e04778bd595625c4fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124607.22/warc/218c6f875f454bd9d89c4649b5af97fa7b47d047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124607.22/warc/5b4eae76ec058a0251fb9ec8db1a93a5d3c4ba3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124771.92/warc/b7eccd74fc4de428c9e91431cce4a4c7b0179dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124771.92/warc/c907743cbf4c7bec04a08b5159c11eab47d43595.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657124771.92/warc/fcabf575c5b4b1f70d00b989b1d475b11d0749ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125113.78/warc/ed893b25afea0cc4c36a5455a86fac131fdc1f78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125113.78/warc/f9a0b15a4c514e69306680f96d190a7f53180b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125488.38/warc/373de784bbaf098315946a907adcd2380c8d41f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125488.38/warc/bb7f152588d85e455c23756adf7f5e98001c7eec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125488.38/warc/f0eb024d1fa0f79382b8627a6fce203b5c719c79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125654.84/warc/7b539086087947fa3804a4bb3bcc0a72c5743329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657125654.84/warc/f5becd443d037ef4839aa3559aa1ceea8c581b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657126053.45/warc/600a39a08ac88fb19e47f555464bd7e0b2a13ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657126053.45/warc/cb2a5437a2aeacb16eb1ee6ac59da00de1f23f67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657126053.45/warc/f8b7bdb69166275d6aa6b933affe1fcb2a0b8633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127222.34/warc/0768518e2fdda662d0ed4e9e254010a4d940d4f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127222.34/warc/fdbb38e0d3aec4b0cae4f363a09effbe642bbcbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127285.44/warc/11381dad6de3834e90aa556b396a265732ad0297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127285.44/warc/7e8a7bc21482413a36d8a1eaef0e3e3551a0105b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127285.44/warc/d783efda50e9f4c7099b883c22bfcd319e75bcaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127503.54/warc/36046d9719b7ed1ea175c4fc45e2cd7ecbd4c212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657127503.54/warc/a0d111aac1c04de96b07a20b1a5e9b71b157cd57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657128304.55/warc/1c5b00699db5c3c408176819432d6dd78a3474cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657128304.55/warc/354eee8c748e19347fe1ad8a042811573d881db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657128304.55/warc/c5e54a7d1c2c36c43d14c1d07c6597e8a1fbcb61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657128337.85/warc/4331492cd59ffa3724bc7f6ad6e2b38ba74e25fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657128337.85/warc/5bace44f5ee288ffe0eb8bf5cdb40c24480d535d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129229.10/warc/7f5a4b709c8d72996d0f0280b9e7aca91b8e2b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129229.10/warc/8a379a3a05a69881478034ab735266012fdc89c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129229.10/warc/c163e8028486c03e1afffbab6117e8eed802e88c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129407.88/warc/3ab4ca13b1f378b916a6c0e006dcbb55bda0346e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129407.88/warc/af53e416b259da6e66adc62061b96a878943f954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129409.8/warc/446766bd09f09eff0e00538dd03f240cf7ea873d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129409.8/warc/70479fdfc00932779e958a64374f46979f958f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129409.8/warc/a5e66bb983c64df42d5a8f577cee4316fd518f49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129431.12/warc/095882b7f43847f340cee8b58645388dbb3f6e8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657129431.12/warc/4abe18b4f06b3ebf641c102fd0f5185a2aa84ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657130067.43/warc/1ddcb12e070307b093c4896cb10bd5aa7c392a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657130067.43/warc/33ecfa6d1d524bbb50e8c88028e4fec3323ede28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657130067.43/warc/e54bc6a295111cfe9cce0b062e3c68baf0298469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657130272.73/warc/793b469b5c6b6f3936f3433314491216ffa56ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657130272.73/warc/af029c90c7c618a30330537d6bd092a5fbf2bc9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131145.0/warc/0975abb49dc06db1206cc9b5f338147cb1ad6ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131145.0/warc/9753be172e6681611b829f12da0fa55540b0e02b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131145.0/warc/e1393e0704333a5b32857f1f6df067b9f3cfadcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131238.51/warc/15e38e80eda3f9a14dd07cb7e7816810cd12f3a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131238.51/warc/e8df21c3d3831cd8def97f3c6435f79ed2dec9ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131304.74/warc/6d1650e464c50757c77b1376295242caf829fd72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131304.74/warc/dd76ccd353b83f432fbaa27181ff5ace59c9fa1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131304.74/warc/e786b70f060a52822c73043b49f2ada1cea96351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131376.7/warc/06365deeefd0ecb028380890a16e9fb77cb70052.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131376.7/warc/2e601515dc982eca4fd7082a1450594d8e9c67a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131545.81/warc/6b9c78e2f45ab919936af990d6a219207e8ee283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131545.81/warc/9c2386088390c903101b8ae56e0be4449e3280b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131545.81/warc/f3d8481eb0457646dd7ffbd81dec1ccf122a9b13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131846.69/warc/4d8d500125cded777ec2d0b22e6ccdbbf16bb471.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657131846.69/warc/6ac441f92ec381cc31299f27bc8bf7a6bd4ad84a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132007.18/warc/368567f72dae83cd57ae00bc8e17b9fbaac1c2d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132007.18/warc/5ff11441137a368f10445b8cef83a0b2a5784744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132007.18/warc/d9b1c1c4474fdda95ca6c55df2dd390dfc017069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132025.91/warc/5a7c0053f218752a8e702a747a179696e33a1561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132025.91/warc/d263d738e4093ca2faa364e6ae5de558d773c064.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132372.31/warc/3d0f938af23924fc5cb64e7ab7876374c46dce2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132372.31/warc/dffc103830f37aa9377cd5367cc351779dadce48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132495.49/warc/55a4187d4f3e02d1c4233d0fc8268138cac90947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132495.49/warc/57e57c29214724388bd3eabcc74d60a797a351c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132495.49/warc/763256511a755f7597dfea8b499db6366785c13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132646.40/warc/83e29f0825df8fa36609d1c97b93530eff2b6a04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132646.40/warc/b60669d4e2a49212237619fffd72fd88ad2bc027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132883.65/warc/1a629e28b8d269de9d8b90573a6a64df1bc1205e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132883.65/warc/b7c132f27b2fbf820f8c7d89f3882bd7b9230da7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657132883.65/warc/cbdb1ca916bb4b6de2247c75d4f12d0990eeaf1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133033.29/warc/2f7e91dfdf77c590be42d2af92f06d8ac500a393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133033.29/warc/f9fe3e4651876a8e91016be63b5c739ca96f06ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133078.21/warc/2d29b152fdb342d40589e8f598d9e6941bb8f0fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133078.21/warc/6f30ceeb4c574e019aa680db017728649a565451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133078.21/warc/914a342f6c1b3a9c0e22001d65f32af38cde9444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133132.72/warc/d534c387836e7a46f63af86fdf29a0c0b53d4e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133132.72/warc/d9b6e796ea825b47a790dedea52b558f8b9fdfd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133417.25/warc/029c1d50a11436bfad51fa3bb09d97f1d1bea104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133417.25/warc/2c17b315b8ddcfcf47e293ac4be2576b23c6f883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133417.25/warc/a2d1db0bef4d6f22e0b912129ad6a74fb81bdf74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133455.98/warc/171f2c2516c6d3f2409fa24d5ec026643a437d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133455.98/warc/d44a009c5e19ff5eb7dee4c94acab22660a2bad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133485.50/warc/9428fb9ffa058383bfb72965ddbfbd6b6fdf888a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133485.50/warc/bb46403da8560a4f164530aab7c08e2ef1f489bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133485.50/warc/bc6330f1220a6e70ded81c3214c4f671547915ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133564.63/warc/c40e9f32b81893253c07f158985faaa775579fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133564.63/warc/dc3d87c7bfaa9eedb7609f17b604223097dcfc4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133568.71/warc/3025653a235220a0a76f8d2a5df04415bf67b96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133568.71/warc/428db30f7084f7a485d03e5046be770b948b3125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657133568.71/warc/9b36affcb9f94b3e6517f8839de01a595a7ce7cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134114.26/warc/77943a490945d561c30c8f1b55aa19f08e855069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134114.26/warc/eed8329ac33c938f8ab6e31595f9f01ebf900c09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134511.2/warc/158f73637a2004e0814db72b6e3c4ce1d593e798.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134511.2/warc/1af7ba976ae5328d7cd78be9305ba506c2697846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134511.2/warc/40b2fa0cd5ec6fee7e54ac999782b605cc594808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134514.93/warc/6236cb158e840975a6d78ad393dcbadea264e927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134514.93/warc/bb7eaf7154ca443f1ffc19763afe044e81507b96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134620.5/warc/0f51a47a03a2b814dd73059694cbc527a653c60c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134620.5/warc/47279467462fa57bfdd9646bd9cb50ec30beab78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657134620.5/warc/e31be5cec4750f416905e3c46c2cc6aeec246f1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135080.9/warc/62bfc5bdaf87b10cfa46ef754bd418c4712938a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135080.9/warc/bc805f973ec8d894aff555c68252b90a3ea69863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135396.68/warc/30fc9165365a48f34db311b590f941b75e9414fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135396.68/warc/88deed70c8e51ee03b87c5488b62640f2bf90c00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135396.68/warc/ca3a839dfa6585dff1d266f50bb25258c6c899dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135549.24/warc/817d488ee85cbc37d52177731d9264fac22901aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135549.24/warc/f7472f987fbcb5527e1775e2ac1c08212ef71eda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135558.82/warc/0705ea6c796cc9870506b75a33bad47766595dfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135558.82/warc/2d309eecec397c3f4daa8a85d1d7e288e082e9c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135558.82/warc/889fcee3ca24fd784d909ba4aacc2a44b02d5a53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135597.56/warc/aa8f69940dacb99d0079a4b9fd1ab7331ce7b81c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135597.56/warc/de3b852c40a6dc0695e0ee8fe282c58d65219976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135777.13/warc/0cac2e2d980b74d0f44246182ee3e7075d7be38d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135777.13/warc/2a94c7e4e588156eac676ccc2ae4cb78f367feae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135777.13/warc/e94374430db2fe03013df2f04aedc56e68889639.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135930.79/warc/38abc858d17a1164af2ce6dd3fbb76e83fcdc0fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657135930.79/warc/8373d3ca579a20ab5a794c018c606c0978b4a509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136181.32/warc/71c53a8e221a14fd2fae028650ddf029e767378d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136181.32/warc/9061617a7966faa03f7456de7ce374d8f8742641.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136181.32/warc/d4ae869852d4a31c9e21258912c9ff6cebdd9e2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136494.66/warc/391293438efe4b5458281ed4e94241b773429792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136494.66/warc/d3697fd383c1d74ca2efca56122eafd5c7e339f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136523.61/warc/159c8474c04a27e6f37e2732ef0edb0df15041bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136523.61/warc/a3c2a152f2f2e50dfc7b7270ce04b98a67a0a793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136523.61/warc/a5a93927adae48ca1867dd3525e43bb6394f164c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136545.14/warc/c10b42f39818f5057b3f6a54f71a413cd4a05802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136545.14/warc/dabc3dd80132b0e6218c9743630ebdbf60025896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136896.39/warc/3e92039d642848c8a68840e754aa791484b93554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136896.39/warc/4691e149b58e7b3f01ac3af11420ec0282ac0e0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136896.39/warc/a828bfdccdab7ac6c18216375e3e1f728082b4a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136963.94/warc/3718026c2dfdacac20e60405178de81267422675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136963.94/warc/df367f129961b79b3875f8efe92e1f0af12a7587.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136966.6/warc/728597696f69b0bff6f54de1c732296f1bb019c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136966.6/warc/c1aa66aab540a52120059de0a1dae27617ee25f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657136966.6/warc/ca45e64ebe701bf7aee672b78f7d26985dba7f99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137046.16/warc/6d19754c4a6af82cb5888c319c91e0b13b611f47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137046.16/warc/cff0da72941181fdbd640538d304df87f4188e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137056.60/warc/2b07d9ee6be885eb7ad3aa4c88bd175905e678a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137056.60/warc/386ad023c3fcb6d6055e1b92c8d173cfc6547117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137056.60/warc/5dcb272e5373ee673ca41b13d630930acaf3cbbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137108.99/warc/4bd32daad8523a34891c3f8c2544aa31dd50d0bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137108.99/warc/96a3687dfe7b4b3d14c574526bdc6b5d7f412a96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137145.1/warc/3b50264c652fbce194863d4413410d3f324ac578.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137145.1/warc/b08994588e07968ec78b47c8126b493d04d7b9e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137145.1/warc/cd90a87d1935143717050f4e3f53112c888e82e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137190.70/warc/1e66301ab3d188f7661c65cce81eb21ae5c3394a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137190.70/warc/d9433cf9c406a40855429d471ff42e3dfa7ca7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137698.52/warc/469c4d064a8f90e6db17be80e9461381ee85be91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137698.52/warc/677f260f897b9d6e4c4a060532965419ff9f8c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137698.52/warc/76f201209a250e3ed36d7437ddd3933eb06594f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137841.57/warc/24555d23d20f7a0e6976acb7cc321b94fea76a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137841.57/warc/cd5c69892dac76203bb2962f6d50ad5b3be8d305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137906.42/warc/1e7661daa57bae6a454098ef6234f28932d03821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137906.42/warc/3494105f1c1b57f00478d5f7a988e47e98ab8f1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137906.42/warc/c079cd053fad25d924088713e8e65d70d86b3c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137948.80/warc/7401132ad513a82a49a32e88dc4e58e8552d8252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657137948.80/warc/d238d76dbd971728dc7f2865fb83adf34daa8319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138017.47/warc/bfcfd1b74d5f83a6fed01ab17d0d13048c4ddb07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138017.47/warc/e8c0356402e35431c9353ea505bbd19e63e3d165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138017.47/warc/f99678777c74170ae09448cc501439da26a08da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138086.23/warc/6f19c6ba8f70b7e2929df2f29b17def771701f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138086.23/warc/dcbf403a5e02a3b4ea50069ca85b5cb6c3342d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138501.67/warc/9234294269cdff241e76ee326d75e10e2ae55a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138501.67/warc/a5dd89c79a7da0333bfde06042a7a3c783813dae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138501.67/warc/db3dedab21510eeebc59c47e5fe8a714edc9d259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138770.35/warc/5cfb46a889be60b6f87c2691f7e4068f36b054c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138770.35/warc/ea289e27b46c8ed7f70c556a2d00834a671bc4b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138980.37/warc/072cbe63aafcc38629fd3fc11731cdf37438e57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138980.37/warc/6dc8913bd9f3eebd5cf65b8486de3012a2ab2cad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657138980.37/warc/8da089afd28ebf7bcbda268919698144b0f793e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139314.4/warc/555dda870c1800f15e544cb0d8ec2283631a1c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139314.4/warc/8337b814944ea143707baeee561c87708f333cd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139395.59/warc/5d39b9a83112768a186e6b9fccfccf8ec32168d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139395.59/warc/d619909315c72cc3b867e2558441b4f19514e6db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139395.59/warc/ef436520de5264e8cb94a4b70b93edffa74d4ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139669.58/warc/802cfb16aed263bc51bdd2d78afd1a6748bd0efe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657139669.58/warc/8e802c57eb056043684264fe0d25537287c316a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140379.3/warc/8d5b5b2805a0eeac726e63674306c7dffb613a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140379.3/warc/9e3506e58ba2dd3a81856712978e203d6e9ee916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140379.3/warc/d27b146f8dda25f9a918830c5137f13b0f4566ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140890.97/warc/4bc27e994ca0da0b5aeaada9085d1530ac4affad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140890.97/warc/920a422b4c60c873b2f81ec68bfd8b527a02bf9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657140890.97/warc/f9800813db6308ec0ea6a80dee6e9e5a127fe752.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657141651.17/warc/6e06f822a74f5df140844c5c542152faad03cfe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410657141651.17/warc/bb1fdbef5f8e8beac69389a6856e2595c937046d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410665301782.61/warc/4e221335d8002f9d0a24de9c341d6d9dac8f612a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410665301782.61/warc/71bd136e5de8551362222b8b920a3744e7caae3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1410665301782.61/warc/fa86dabd1e35a888fd1a507061457baef74b4bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662880.10/warc/2b12f17cb053975164479264080ef57bd4564a5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662880.10/warc/67bc873d7469316088426e59fa03fb74d4739ff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662880.10/warc/79c4ce8eb481ad16f8872ae4956a06088d494d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662880.10/warc/948a8af01405de5f27a0af6b60a8740b5b42d324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662882.4/warc/0a549ece1e450dee0f31f326f17051fb4f8fe3ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662882.4/warc/129f567801602a708147c385ae9a700bb3d486ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662882.4/warc/4a32976c3483a19c0fb26a67a0cd0e882cd34a32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662882.4/warc/e854802960f69cea0546bb349cd6f2194c555b88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662910.19/warc/a7857c36453d00c9a456ae5bd7ac24e42485f6b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662910.19/warc/b025396017659a984bb98bcdc23238a74a4061ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662910.19/warc/e55a7bf46348d0b7d23b436bacc7d3e65ade0be7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037662910.19/warc/f9475fa898f2aca2aa8679729dcc1cf112e7daa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663007.13/warc/117e6a603bcf7b1a6364342fb3b88efabc03af57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663007.13/warc/45daf078f7c5d5318f1ae058d861f342cd6c00fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663007.13/warc/91945507fa110d6e3df2d4b5c9d07f2af8829130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663007.13/warc/93bc220619f300e19dba7532ff5a0391470a4a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663036.27/warc/1e3280480faef91b7bacfd29b5217cad3944615d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663036.27/warc/24271c02e3f6cc84613fe0223d3852ce36663fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663036.27/warc/6b83b4c10c227d0cee30e898a145fbb04c1f4ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663036.27/warc/a87e41b399b2975883ff856bedb7ed79eb56139d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663060.18/warc/43c0e3cc075c49e0a75d6f0a249dcbe21d152fcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663060.18/warc/5f5f8b48686ffdda499b4031091437a9e1c043bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663060.18/warc/9fd1fd1644c6bfb8e868d3e9ff1c8df5eed524de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663060.18/warc/bf9d4ee08989507cea168a7850931985c0224881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663135.34/warc/29f52180aea16d8a2e1c9228a14371be8530f241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663135.34/warc/961eed81694c4b0a74d47ef56d233caa6ede5bc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663135.34/warc/bcc4bfc35fb954caf07fa6af54a886b720727c3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663135.34/warc/f918bb769a5e95ec86959dd0183d373427e67d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663167.8/warc/03df8d7dcdb752c3627d63f3464af1368a8adcf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663167.8/warc/0abb2d1b3d92be0938246f08b5d0d101acab09a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663167.8/warc/94fa2de8184f036a88354549f2358b8e5df5cd27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663167.8/warc/df74b58c95306239bb9e35c555de9f5cb726aa86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663218.28/warc/47cdde28a90e38ea8d47247bb4c59ebc46e56efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663218.28/warc/75958164fc60f426b2b6fdfc9dbb61ee74163ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663218.28/warc/8efcec34fd84eeecdbc135cf4cc3e83d82941434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663218.28/warc/ad229897067fb25e61d5fcb0c872cb8a32387004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663266.24/warc/f9142e8d44ed6e94299f40f4de3bb5555f771afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663359.6/warc/1ff569e16bc74f4e405b2badda0dcc18109e9dc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663359.6/warc/33c1d050959e5aee3d980d075a3bf85cc2fe5c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663359.6/warc/93ee92debadb515b84542bfb2993181ebf5aa0a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663359.6/warc/b6ff6d02355d65f7b0e65f1748965709c696a16c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663365.9/warc/22c4b33d79f71f9ecde530aa1978e743e41e8011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663365.9/warc/25c26c6ae3907248f0928247d62e2c5886758afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663365.9/warc/494c5aa18b8310d76e66fbe77fd35c0ea920cec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663365.9/warc/5fe432632abcdcdf97d1f4d8f0aff3ca2fe76ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663372.35/warc/0315be229b2f32cc72e33c8ad1ec4f6178dafec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663372.35/warc/0418931b19234960663fb4bdd70f2b79eaa9e8c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663372.35/warc/30ec63045caf7f3b231bb1e33f999a9736f94f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663372.35/warc/ded2e06488f39187fa9284efe3be99aa07b92da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.12/warc/4d68fa956526d1aa9fb837d451967636636870c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.12/warc/5da6b9e5769ee191236017f6ea9ca0e4c7ba0897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.12/warc/9b6933f46375c502e542ce45aa8bbcf8498acd13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.12/warc/ef003f2c5988f911f55ec901d107c2582e3936f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.16/warc/5d850a297c0a5deafeaa7da24da843797d3010b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.16/warc/ac63b0f2a77ab3c07390d6c4d415a72b0a469ce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.16/warc/c16d5f589d400c9a9a3eb7947015578a835d0abd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663417.16/warc/cd9954a1ca5a7e1d10c9573bc6db7a9de8003c2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663460.43/warc/364ec1fc88e9c50c7189bf8d58b67b07488b02c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663460.43/warc/93bdcf6ab338073774bb07aea62183ad7317fd95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663460.43/warc/a4193ec9babd6d068e040eaba3545624859d727e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663460.43/warc/e4c2ca2c543c2b389f7d7c57f807fd25494c758d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663467.44/warc/0c7505086074dc2a223074e5e83105efd02b8fc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663467.44/warc/4b9bb9787dff43b0cc2012a2b3e8c831371ffbe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663467.44/warc/a5d46423cd9e86363b5af965ee5f6b0cb5b91d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663467.44/warc/c3d7cdebae47e1bb6c8358051cfe84d4d1fe17e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663551.47/warc/1921d7d0ac2b9bbbde3fdf98012602b2de668acd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663551.47/warc/a2c91777940f92fcd58d1699367e2c4d5a36a516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663551.47/warc/b22058a1bb238a19ba9941802fc6a819d4164fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663551.47/warc/d5c7ea9ad8746fdf7d9b78e8d0032d3aec61cf12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663587.40/warc/856ae3fec50421741b8b7b8d2e2c6bd0224eca48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663587.40/warc/a4ee154ffd91f84e3b68c2034ec074a49df9ec33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663587.40/warc/cc1855d7ba2d1374f64441e656e1552a41aa7e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663587.40/warc/e0902b5cadffc842f635122a3c9cfc3d14e8a79c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663611.15/warc/1cca453cb84f2a291b13c916c2d9dd76efe385d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663611.15/warc/2c7fb3a75a839c2aa035c0b788276e8c8e2848a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663611.15/warc/574284c0102d40b4f691f7ab47be6b7a9f8145f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663611.15/warc/cbc0ac6c6dfa6a4838ea2e546e1878d3daa1881d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663612.31/warc/453c5ed46bedb1a17b9d96fa2f2f28245cc00120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663612.31/warc/d5bd0a7b384b6007f85343df60c3b6ac6f499de0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663612.31/warc/dfcccaa1264f1adf65bdc38d54cd7884d4bbfc03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663612.31/warc/f6302290914b58e8ded144bdf67dbd8a78c249dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663637.20/warc/425b2ee015daa333a4218223dcbd7e1cead42031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663637.20/warc/6c1aa63ee88fcca6049601f833426c98ba480359.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663637.20/warc/b24a002be2ae7f7a6eae8efa3bf02acd2f227a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663637.20/warc/c359d4020239e124c4099b0e1f43dc23828b0c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663711.39/warc/03f72d63b9937fce099712cb490968520c4c65cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663711.39/warc/35d37b025835185a763bbd936de276a88688070d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663711.39/warc/4b703bf23116a4455cd202ed6477ff5db9675c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663711.39/warc/b74e6eaee57c886d8507c871036a8e6d4525359e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663718.7/warc/3b15a170c18b957994733f7bfce962444c5a8a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663718.7/warc/4742b6e31362f98a9abc22d2257812bcebfa171d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663718.7/warc/acacf05e03ca5b6eb70719ff01ac811911bcb0a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663718.7/warc/dfa85961a82e77b959896554dc36152d08655bd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663739.33/warc/65da1ae26c94e072597d68925cbe39a2c0f4b9ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663739.33/warc/6ad320a04d518c11c01fb95b3a54f297b44d3327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663739.33/warc/7ccbca3656174772e8bfe8f5619a5d808d8466dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663739.33/warc/f5a6305b1ce114da8d05bf42129853b140e0fcd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663743.38/warc/65c3a76026757e00427c35120c1646a0e834c786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663743.38/warc/6955767cee9332f87a5c54e4e64a7b53cb462123.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663743.38/warc/695754b3369c2b5f757402e6f4b48b8846d00651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663743.38/warc/f87ac50f0008301b999d13003999d0444aba07fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663754.0/warc/35f5cfb102b5de5c1d2cb94107a25ec3bd4c8463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663754.0/warc/8f09c1b78e061b034c3e2f219ce4c6a9ef482f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663754.0/warc/c2daf295df0d24c4e0592b907adc443935c63941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1412037663754.0/warc/e63650b653b799cdd6e61d0c0c022b313993ebdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/17c4492eb437e735c1b3c0a45454ce263d9173d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1927792eb82ec72d4e8f3a2abe8a73e198d6823b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1e76151b3cf30171fc0603aee1b4f01121c7d009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/1f8621708d3e2b844718cb2c5152f8e46159f21f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2264485c732aac13c59b989bd68a534b16967f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/24f0f1bee90e236358737e462fad78d6e5cec37f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2663989d4a674fd87fda731c6d2736a1d339e7a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/268f9fb8a5fbf57171c10f98cfcc753c4cdfe8eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/269905fdfd007524479d4a3aefa2911aab59194f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/28b161b3f7b8f1310414df2b8c15aeaf66413e68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2a2a906d2b353ce65e4af18e891fc3c3048dec88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2a329af1cecdf56fcc44aedf985399378ee97ba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2aee3c279a1b6f678105157f5459a0972220170c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2b2a0718cf6901e3087deb54043742d00a8d2f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2b63e6da9ddd7aaaefe12382456336bc8682a0ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2ca2f9448c9ed913ab2d0e060e4dd24937c103b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/2d706cdf93311d2dc6f3b0f7bc042caed49b6a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/35f7122fd804719ca1587b3ff3cf756c6502a7c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/39f9bff3e38141f80e89f44fef91066ff93c7937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/3c56fff984830177fe4e6e7d84619368af0c508c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/3d98dea0134d024c3c1eee15ddd9de86943b133e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/407fdb5878b3ff0342328705a3e9f5cdc5b55323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/471a11d220ad4d9bc60e32ae1ac70f0afd88f3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/48cbfe503fe26c610806299eb0d3340ac0d7d728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/49022b8323df954a81e70a6a37c1b3af8a365956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/4b8c6a3b2b99b36c90f25ae7e3b5ac6db94fa59c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/4bafeb7bf640a60037aeccd354d5aa5942ee6ba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/4c59c0d1e6dd571885b50ec39e8d0605b2495b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/4d243ec6d1e71533b8db262a32f2085795d75dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/4fe9491ee537ff35b5473a1b7218d2fb63382f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/53571da664d482f426f3570513ac2a6f2746cef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/54ee32c65f2996d98a5be06654bc3b928436978a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/55d66f6714b916357f5339e6b8117630b8e97935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/5664e30af8b1e0320c4a44430c2044a94ba12a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/58924a921c13ff742a1e7c9d453c091f3025d74f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/59c42341b23e67fdad9fb3d7b684b93157f94e88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/5b32964ca3e1df26c1225aa4614b6f96f1932039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/5d75b0005e870604af19cc143be5554f959531a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/61aa1ef1af3363033c8047145399a2231cc0625e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/622abed095487a7aaa0e4030c065e8d6e554ee5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6427deaf7f85dc0eee031ea77e69aaf12c26a7ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/668839477d44e86aa2df50d78d71cb18f70a33bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/66a43d715b9b2df21444423def0f687585d41218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/68995d32f8e6f3116917563d5b57a68ceeea946d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/69805f316991a63d8f927a2d0b879b8f828d7193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6a106dc2dec22a6c39f651f062e85ea0edca76ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6aa56b8508c15d42c7b5506105102646f0fd41a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6b7ec067dbb5618d50315fca3a1a1bd8c858de8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6cb0f305dc88777f57227dee1b5c605972558e6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/6cc2c3baf86fedc38e86f109bfb70f49ccf3c390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/72b5681a5c5ab60f3b2ee53bd6aad5a23f7603dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/75c07c4e0447038fc2dc2a0d6d878cd8140979bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/7d7d6707efdbeb35dfdda06e9c614f546b0f2a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/7f476a25bf2d06ea0c7e7df414219046e819ff63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8114a1ffa74eae86fa0bd97a0c8f66050c8f19ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8128a469cfc5d15459012b3f809d27bfc65c348c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/815d37b0191dc7c58c405c0a55c2cf79f7109547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/83298241e8ea9010e9a7daaaba6471534cdb0083.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/841c261cd8d18d486d4a1f0eb1b1fc66706a3892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8737b08f3fd3b8c409845d74553675dcea772c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8ac16690a84ea1a36383590957ec24b3da165522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8ad13fec2c88568807d3cc5faf21557bea82f28c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8ae61fe35b0d5cd31d692bac123f4ab8c7043137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8b213033322c0f71f1676a286a073ad62778404d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/8d4abec8880b53ee0a64725b4d22331328d3ea8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/910ac374ac816e9b00c86a6147923348e00708ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/91568bec38a419a4013345648fe1c9318a993d41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/938dc9e5addbc0020de47242a515af7151935c35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/98170280bbf212d418ff18e12cb71de28000ee1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/988c358c9fa70b26faca446c58922170a8411963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/9e4d2e024bc3bf24fb171264619c8771094172ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/a5857593d016d7ee7bb7212ae5cc9682762b6eeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/aace20df63ba8ee36eada6b52286c789afec186d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ab0af9c583617a36061c10aaaf2996ccc7fd17ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/aba72a5c19fcae9a39c21e5cb1c00d9e4322e123.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ad0935a362a9098ac9bcce26cff8215823ef6e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ae5ac17ab19c3cbd4d7fa3731e4e0c47a08f8a3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ae943bcf1350bb614ed59275ac5d1040f0b14f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/aff50c8f98ad2a4553c121a31619298f5794d100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/b2c9fb9f52cee7578fa7e1fa7066b3f738e21395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/b35cf4ad8f390dac6ae9f8bc207e64049c069bb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/b57c6fa416eb9f8942c7173a18a9c965aad105d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ba2d440c774a8ae184ebfa109405de77f99d480e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/bd0e2aaecbe69dc1cbd8b84e5e12dd1d41815ccd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/be49313df1f5899bb914c67b19a2fb358fcc5dcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/c0ecc1b296a3046f700f697ee206310419d809ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/c3fc729b0d38158dafc42e5d27e6df13ff18764e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/c5a6b3b054c37b575f3734493234233c6c17aa8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/c5d44ae880fade5133b25a11982cdef305ba77e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/c681e557362f2c5f1143737ae9831956f070d9c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ca1f17da70b395f7b31f7c9596743011f99a40e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ca7140bc273641e3e489dd96df500a3a464ffc23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/cbacbbc4c3e3e7a57bb207353d98804e6cdcfd36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/cbdaaa15aeb3f4fa8b452cc8a59fecf2a62ad42a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/cee076de996aee076c3148907fb6a0ecd67566d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/d1e9dab63aa37fd89d4c5dd65922f41addecc82b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/d82b70c1e6ef442efd420eab1927630a1ce904a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/dc3b4743c194a495bfacf63b524350b5f21d9c5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ddd7f01b6d12ad609cc2957d891374b7e27393b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/e02f8c5ead223fa4beef0db58b641d7aa943a424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/e19e838220a182afed247cf1a0c3561b638c9f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/e5a14e2efd62217bb3144a0a34be506ff5152c51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ea62458c6b3e7e7d7024971f2434fb7cee4a7942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/eae170149fd8667f9b18607a720c171a629fcd13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/ed2b074e56ea0840600e30a57b0c0b74aee11690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f75068845c8c32144465e0f130fe14bcca949dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f7d704fc769ed08dea02d5066c70b124ef2bf940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f8285ffb90ef0b6eb4884b3e8a5971cb726297c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f861ec0be7988c043ffdbdc12613c907939606c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f8cfffb70923b69f7cab2a4c06a1719e81c6d78f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/f9d178d384f0a41bc0c77d19f93d134bd2f936e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/fad76d756e87ab9ab24b2c89ab40083002417c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/fb188f2b42bd26ca3fb000db20ea339471de111b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/fd432cf6b52e977ab30b4212d536843025993a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/fde57dacecaea7b9c05210ba987324690efc99af.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-41 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-41/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-41/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2014-42.yaml b/configs/cccc/mixer/CC-MAIN-2014-42.yaml new file mode 100644 index 00000000..81bf93a0 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2014-42.yaml @@ -0,0 +1,706 @@ + +streams: + - name: cccc-CC-MAIN-2014-42 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/00383f6644591eb0f141cc1a2e0daf59063db031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/02c4908dea8566051c2d589c053e4de73c86121b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/031663dccbc175f518a22aace7e804f106f207de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/042260e59f7707854dd8a43e820793b2d5c28c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/043aa39674bdd810bc208679bbefec60b5ff262e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/051a7a3c8ae27679f46fcd648c108799c9358396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/0a7474b9848daf435d6660b539a915fbe87dae75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/0b3218215cc0d5e6c94fce27fec50323f94efb39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/10ffe3152a1100343b17aa72df5b285b3477452b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507441801.11/warc/b4718135a15b2518f1f001746e62536288ea2f1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507441801.11/warc/da456a8b385bd6434dbbff666125b8ffd7972d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507441801.11/warc/e123ed048f9593cc2daa69f817c6cda60718a8d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442288.9/warc/94629f3fe8943ce464c09861f311ed60d1813011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442288.9/warc/a803d3d1d8664d02d614cb580304da56efdbe156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442288.9/warc/df383a2a89d304e59575c6a7e7a33943a9beafd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442420.22/warc/7aa5e55f6f80313e6140c993a1754f6c0c0f2d2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442420.22/warc/90716faf413ba38e8253731b94537927462ab4e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442420.22/warc/f3235b9abc051018107c10af861ad4fc8476956c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442497.30/warc/0bd3389c620695a9d4887e67a45b7768edb9dfca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442497.30/warc/1985cc66ed08dbfbf9d56bd620ab0e5325696022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442497.30/warc/d4eaf9a1d42aeb3d070b61faa79d83f612f73857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442900.2/warc/07ab2979df5f506ae6b8c3819b7345ac6a891b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442900.2/warc/f268fd71ae5081bfc6b72e4c3697de61a6143571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507442900.2/warc/f8904143eb83e7ab0fba3b87dd4f51382996f6c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443062.21/warc/6970ae59833879af16ed214325215a414ccbcd15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443062.21/warc/9c7117a9329f97a3321781b939b2d753f43c1347.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443280.18/warc/36d4bb36e60ec3ff4e1cdc625f46af7efc97238b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443280.18/warc/70d92d272e125871567e4614566f2325bd0ac672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443280.18/warc/b235d25def52b2025458941c1d4f60d6a2b11fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443438.42/warc/dc69680fc667f576e0eca6f45bdba31457b1d89d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443438.42/warc/e75bd86a7b8ebffa3152fd04f2d77edad93b7b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443438.42/warc/ffa00dadb492442424a488a2af7d70d159f273aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443451.12/warc/0adfba1928bae9f5eaae6e0f0e595233f9c8e963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443451.12/warc/67cc2748d98fb106a306a86a84b2ca7f224558dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443451.12/warc/d6e78e5a2279183ab501f9b20a969af8c51bfbdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443598.37/warc/29bfb2b294c5dff5b4a9ddd7d0b4575a6b1da602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443598.37/warc/2a652ecadd47c583379b2194c573a58ab0f90d24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443598.37/warc/5b59d06b8cb9d46231d3d41e2461cbc98853e1b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443869.1/warc/12f2e1feceed30b85420c7f40b93889ecc6d6b2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443869.1/warc/ff93ddcf86b75ff799074d568dbaaab4d2484084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443883.29/warc/4ef0161c199230d94106d8c7fd20adac765c8911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443883.29/warc/6937e37c25a366ff7e329919b389f7fd4e70871b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443883.29/warc/c438974416658a0d22df86aa6c8e6af8df329326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443901.32/warc/2e8a8aeff60296ac6d9856083692dacec24a9f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443901.32/warc/5b3b76f82444fc3b54440726ff8372416b6bfb76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507443901.32/warc/820a765d475ce7a191ddea47714b3a3c7f4d4270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444209.14/warc/33e98db944692b6d030629cb5fd2e7fb721145a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444209.14/warc/5672fdf111645de127cceb01d631deada468b086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444209.14/warc/9046549d5e21431ed8903ab14bc2ea9ae149deca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444312.8/warc/04a8b60c56b8bedcbc55c5cdabdf2a9d84429791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444312.8/warc/36fcfd3b8a549d1e6ec2d125e1fbe47321469942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444312.8/warc/ff2064b718b6ba4f2d787ecde2cbecf41652115c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444339.31/warc/26fee23923969c287847c1027f0a45575b0eb35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444339.31/warc/ed912e546868a2feaf51b915b4c6bdb01ed833c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444385.33/warc/14379a01f7880e31f94eafc90e291a24610c0f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444385.33/warc/7b262c0e5610c0cd911c6b37ece2fa13596de983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444385.33/warc/f72080724e1537d93ad9d8b6b6c4379685408d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444465.10/warc/31d925ac201841d1fc408a2583d8842afabb342d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444465.10/warc/d32783a412cbf5f1ed89c21683c58ce154c0f022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444465.10/warc/d5463c2782a674f305a350d81619123e416a2f5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444493.40/warc/07e23c75c1aee48f342f3e6630b6e56de69ec81e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444493.40/warc/17f2b7f2eade4a1e3330596eb510e0a6bc348be0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444493.40/warc/a077df36e6a5690051a0fd6176ae1bf841edadea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444582.16/warc/3138f03a8b6c28219fea592394ff1a2f5c98c8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444582.16/warc/3d22877a3c51beb0a93d8ab2522fc67f1be0cf8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444582.16/warc/c9db86e069e9b10aa62d9efb52a784f28781c309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444657.46/warc/1a8e20662f5313452579fd52b3410a85f794950b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444657.46/warc/aaad847af54b9c922364153e96b5e5d320e3019c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444657.46/warc/c5f27166254176d6807c97b6a8fa71c5391fdea1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444774.49/warc/6703ee86b0082836bc5fe5ea2f3d20619e3c6c53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444774.49/warc/b22b0788f808cebba30996b0f348d7b4eaf94d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444829.13/warc/069c88501d706c2bd91faaa297620f25bdd140ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444829.13/warc/0ab448b13f6088009d1f5d48068e99705de1a9d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507444829.13/warc/8fd49c966d68c7c9ac2f76712536e4306fe963dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445159.36/warc/3b3efedd5ac375f26b8c36e867deff0976f4d56d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445159.36/warc/50c033056de2397287a81edf298d94f21274615b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445159.36/warc/ed3e6a5d32736c4587348ff937b21ed62b9ea9ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445190.43/warc/4c9874d5d4dce1050a1a0256ac900ef4c446b337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445190.43/warc/959c9572244cc39f9116462a6f3eafa3b7b7a5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445190.43/warc/fbe5fbcc9b274f1d282612e0592991263835ac32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445299.20/warc/26de752272e28df6c6412814d27bfc5c0a5d5462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445299.20/warc/64de09309e5b530fea5258bf399df84faec4ba33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445299.20/warc/afc906a195968c7e37ba026248d652020c9895f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445886.27/warc/96ebc982f38e4578bf4012a44eb4ac98aae4d44a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507445886.27/warc/e66512002f71994fa23a84b317c761dbcbb2a435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446231.28/warc/2468699e99575129f4886bbfdcdb44f93483177e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446231.28/warc/2e27864e802fac39cace25710d432a0cf1d5d90f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446231.28/warc/f6e26e3e25c10407a479971e928b966cb5351637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446323.3/warc/b070eda75319ca8374c64a0587472468034cbc63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446323.3/warc/b85ce5d028aa322f2de4d7390e0b1c72c8e6a035.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446323.3/warc/ba015ea114139eb8394d4c3a34ae2fcaa77c30a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446525.24/warc/543653fe07997ce2767451708dcf69b7a250a689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446525.24/warc/807961ecdad66f6e3b397eb00dd25df14791bd4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446525.24/warc/9d41b506b15c5175a3df11799a4d4b249cfd5379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446943.4/warc/0186296bfa45c81f5809a5b19604591709ce3952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446943.4/warc/b3034a86bfe9cf64f447d0f3a134edb75cf50f70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507446943.4/warc/d5d9069e57273158785ec9985242492b10bbc117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447020.15/warc/4ed8bf0d71d516b436391032cc3c40c5febfe522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447020.15/warc/9f1a4961fcd53c08701491510c8eae8670320f9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447421.45/warc/6531af2823267295b572de7d9dac177450e4a4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447421.45/warc/9771cc93092c30e62815a91d83404a7336d7c473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447421.45/warc/a382bd132641a60751bd9e8620aa34cffab6927b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447657.38/warc/77f8087fc8aec354ad8b2840281cee8ba7ddd98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447657.38/warc/f0112b20ff4fb16b2410e4a271323f5057007c22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447657.38/warc/f51df20485567037a512ba7a8e4b54bb70cf5534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447660.26/warc/1c371f14dfd59000ddeda4729bd46118f344efcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447660.26/warc/28500886242addba92c55f8410c979121b9da5b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447660.26/warc/399b41922e8e869a75c3ce9ef369563015fec317.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447819.35/warc/0b6ea9a9ab74ad01d62618587c5600899bb2e88c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447819.35/warc/2350a7e49d4f771309332829a87586b899bb285b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507447819.35/warc/73658c8887840add301e73ecdab7dd373966e851.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448169.17/warc/4ad322850c5be82678572d1efcd19315de8e178e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448169.17/warc/bd631fd0c4a0b7ae67b1aa4914a9376041da52db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448218.19/warc/0891de2e007594616610c66f3c4a37e03dcab4a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448218.19/warc/08cfcfef263e54d66413689f009219bd97f33ef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448218.19/warc/7084e3d9ee90e3a891aa50ab05b4bb9a1814e95c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448896.34/warc/3ccdb5837c57a4fe6de419b775e6b73cee30b12f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448896.34/warc/8fd1c8a4af03578e00c8b4afdb439920aede2dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507448896.34/warc/d03033adce071ac62150e637e765efc90c69e56e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449153.0/warc/0a2322086df366afcd4048e2bb44c6144fcc1915.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449153.0/warc/595b8d5ea0c6fdc726378691ce3fbebf6f4129fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449153.0/warc/a731c2d4acdfe524b4b1c6c77dff8345d8de7d38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449615.41/warc/21904a2e4484bf045ef8827075c3f7f48a9359e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449615.41/warc/60b69832bac7d3e978d73a2249b7525fc7b37ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449615.41/warc/64d35838c0b8ee28da2882e2ce33c55d49900f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449999.25/warc/0ba01d89ae0ef106e16bac3832273c5e7f905733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449999.25/warc/41ce0dae3da88ca8ddfedacd0eaae52feb1cf289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507449999.25/warc/d6a6b7eca63b558992c1f42d5507caf3556c3b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450097.39/warc/40a80ee842ad66fde7ea547babe280c7db4e4a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450097.39/warc/e4b48d364d3b6fbc6a9bae477b9a42e6d786f30f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450252.23/warc/466a8d74af9a265a3130e48b490806fbb7982472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450252.23/warc/5ba2e0a4297fa3a4cc676c477dea165fe14baaa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450252.23/warc/91a01427727aeb53342b3b656c666bd630ed635d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450767.7/warc/d1c5187afedeab4ed2bb78d2133b15d96e0cd6e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450767.7/warc/ecedfbeb80f73eb9d8550a3a6689a7d4230509c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507450767.7/warc/f430961f667496bf1e884216c7e8683cb60b89d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452492.48/warc/454e4ef35a2111f6cfecfcac1a6ae6a4344647fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452492.48/warc/b2f5400976955b78fd59b640c117804e4aca4945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452492.48/warc/bf16a0611382eef65bf87bfd14bf6fb5a9eeab88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452681.5/warc/7167f5036eb2509ff7bb8454ba505ffbfcc591be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452681.5/warc/b77b5b89f873e1e3fe89890d8b508056aee3dfed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507452681.5/warc/d575e5d9a862ee3edb5641e64c9953205d3fe8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507454577.44/warc/04888415099c8adfcba15bbbee4ab21bf78a74c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507454577.44/warc/73accc203c71dd6d97fa110b56852c3cdf403fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507457324.47/warc/323a93caa8067e338ed4cdfbba07a40be916576e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507457324.47/warc/567a0429364fbf0eef05509eccba7b8af2244b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413507457324.47/warc/a6e2ae8039b2a2062eec0450e37d87b58c4a16a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413510834349.43/warc/4830338ad8ed0f71a99c27ddbfe7a790735b8a94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413510834349.43/warc/519cb49d76a3e4afaa3b5bdd9d58b4127029466f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413510834349.43/warc/d584a4e7a0585f1a1c4a9031c18cb39674d8c289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558065752.33/warc/59bbf29f5298d80556798054f234c95217b0b3ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558065752.33/warc/6d591c3291d2edcf1ac373ec81482fe14dad64ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558065752.33/warc/89296b21b88bd9ade10b8fa68b7fd6d2bbf85253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066265.80/warc/73d2fa303bc9f09db7176b99185eb0510e0cd87c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066265.80/warc/c5ef00329480c2c5e81cd5f59b58f2bf58d5854d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066265.80/warc/e94d7eb36a138a118ae15545739eaaab87646eb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066290.7/warc/4f9a530d6535add741bc63eda54b42db3f921992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066290.7/warc/98a814ac33d76ad5a83ca29ab9dd44677cfd6195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066650.89/warc/8b9a27779c0172c65d39098c85bde4a2d19c754b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066650.89/warc/9fcb268a886e013abf54ba042338a72a8d523f7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066650.89/warc/ed4ee9b2d24d9bcafb0409b9ce5a5c31793925de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.17/warc/17d790d5abc6449a83d412ab89083e940f0e28b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.17/warc/2bfcf3ae0c39f61b5395adbd0d9d35a80b595dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.17/warc/67834c0105716d56ab6c2d743288de5ec8cf39ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.4/warc/7f75929524ded508dd784ac7e2ded78e5d2eaa1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.4/warc/87b8362d5abe69c78f21da02e9c3e80f6251a6cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066654.4/warc/9c1f7161eb1374a5c863d03709dce0d68f9e188a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066932.32/warc/4859bcd95d9c3c7ce0ce19845004bf824dd5f980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066932.32/warc/cab744ac18dad26bce8c51825b53f4cfb87f35d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558066932.32/warc/f2763d23f1ca9aa58aaca100da76cda672c77869.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067075.78/warc/9f7516cb33e07c030a26b1a8b863592a7ed72d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067075.78/warc/cba6665029dc16bc63afbd45e6da556afc61c996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067077.47/warc/21b11ddfe4f77a3a9bd0d020a1a7cb54dd4faecf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067077.47/warc/657408ff9fae483616bb41a78c64f39904236e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067077.47/warc/da05d3e91060a23970ffa7e1d7dd32061e22472a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067100.54/warc/5da363bfa8e5c27df052d8f272e8585093b1a203.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067100.54/warc/7be0d9a5aec882073ba88a6cb2f928f14aa7477f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067100.54/warc/add0d40db43b643958c3bb5edc613a4ec6dcab0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067114.72/warc/6786ea3131518bd547f3299595c74f0583ad730b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067114.72/warc/99da63d62d3f33dfb9ab3642fed65ca9ec9a99b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067114.72/warc/db154aa185c0625ee223670ed18f7c85212129cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067214.90/warc/17f75aeace45e88885fa67aef09db45256afaf30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067214.90/warc/37540e4a5c8832a0d529745d76ce0299354ec526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067214.90/warc/4622d9447feecca3dc15ba9908734934afc6b5a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067467.26/warc/30b1a50d53cd858cf3a6dd68486422fb95490ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067467.26/warc/45bb5d63b38d927a2699d38dbc523021c41b052d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067467.26/warc/9e794cae36528762c1caad115ed781b81fbb04e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067496.61/warc/f2a39857acddb5e9c703a5e7f64790ed870d3cf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067496.61/warc/f8a7d1c68527b17e05e8799041dea1b04034ad88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067609.27/warc/25befca56aef1578a6929a355d55f0a3eeeebaec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067609.27/warc/bf9abd30b0c1494b6b5e69b4785c286a56388e60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067609.27/warc/f12abc18a77e8c7e42a51e14b0e3ecfe839065d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067648.77/warc/04bafdcb7e4119fd6d6a05d3b39dffda4e9a7538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067648.77/warc/37c2362713d7007eea4448673ac6809f84dd3f02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067648.77/warc/641f0657b4e0140457ca6b143b2a1baddf878381.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067768.10/warc/46ed747d639193fb8e51d631b62a4280beaaf4d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067768.10/warc/9fd89f1f5dff09f36bf5a43a4956fdf23eda55f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1413558067768.10/warc/ce083dddbb6527c2d4b0d3ff460a5c7f98d004d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645329.44/warc/1721a2473bd80517197326d353e98bef2b74ad11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645329.44/warc/b22b78fb25699a7b3e99d8884ae423ec71a59a16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645432.37/warc/095da43cb982de9882859c9f562596e35db03fec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645432.37/warc/474d5858246fce9b00897f909c84b2fdd9ab8cb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645573.31/warc/20ebc3ceab12f254b37e768f66a4c1c0bbfca2c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645573.31/warc/6411abd1f831bb96c872e0759774222661c0dfaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645676.15/warc/9216e9676fa4d4017cec82e0d44701fe0d5da4ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645676.15/warc/c0b2dfe075ba014ed39d94b31873bb9389a04256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645676.15/warc/c8b8effeb01157c9cf9fae14fa558274c16e0ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645845.57/warc/b1d1d1751b61c77e110fa8c3d25c8937e194a541.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645845.57/warc/c9bf07fadaf6073bad15e4521498d87537b2a559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645866.1/warc/05703d25fecf0bd912245c5bc77592a5d0157898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645866.1/warc/782e4bcd6eb63f5ba31f2140e707b6cfba10e751.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645898.42/warc/2bac80da6e1e3259ea09be16f43c365e330aeb2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645898.42/warc/79f522eaebe99812671148b80dcda8afcdfcce9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645920.6/warc/94e327cf27d73654efa5da8bc092c65d293673e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645920.6/warc/f5f77615a8577a91d998b5cc06f219a5d68d718b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119645920.6/warc/fc4eb57839425d6a1b92a9c115bb1009b7e4b324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646008.13/warc/f9c33a97f5275f887e5ad0dbdb4ef3b0263ba42b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646008.13/warc/fa90a76773e81cf621eba1eb8d8d806372070682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646180.24/warc/2af4315e4b120a829142ecbb2c7bea62f82e2196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646180.24/warc/2c56f2649e8f73f0a13abe8a0dc613a9b41ef7e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646209.30/warc/7984b7d7389b28771ebd7b9cd30345251d09d924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646209.30/warc/c49d60054428ff5f8fa659ebc02de42b514442a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646269.50/warc/0fdda3a818269032b30a578f178385a33f00f50b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646269.50/warc/18104a1290acdeb90027c15a2ce376e528297c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646269.50/warc/8024fa5eb963b0e92051c4028beeb1a580320182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646351.12/warc/4fa6ea61f35d88528a56b749193a1997f79c4a3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646351.12/warc/f7c51d46fcf9072ff5dbf5f68778f6087298128f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646352.2/warc/04587bf6675f6dd85ec5c9692315a40a85366b0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646352.2/warc/44f9b5de862adb3b5d1e5073b06f933996804af7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646425.54/warc/19834b48f9553845201fd86aa3b92e8f74cf7acd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646425.54/warc/7e3e14c862ee471c8ecdcf878b36a3eed744c414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646484.16/warc/755d030d8e7b5ff5196c2449543f33521cde00ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646484.16/warc/d6bf4815905899f1ab9c2847f09287a62bc759c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646518.17/warc/2a829cba86572fe94e19ab6d15d778b1ee0ce82a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646518.17/warc/4d5102ab23129a428b44a709ce51d602c219463d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646518.17/warc/a946b85e16a0ab831815316e0a57e48d2d221659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646519.4/warc/3eea5cb1da25ce702526c83f4fbdda43efe79db0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646519.4/warc/8eb77904df23d0884f25aca8d2daa5077b8bc3e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646554.36/warc/089efe7ccb64b663f0df6d66622ee83baaeac1bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646554.36/warc/e3cb9f0a50fa05cd7604032e213809e1b25da879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646849.21/warc/46abc5621f94fad6da78cd66e38fd44aacb45346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646849.21/warc/4e6d980597803ca7e257a34403e5ff246d5380bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646849.7/warc/0095e72a5fdc2e4fc9fac9405391c39b649699c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646849.7/warc/31e72f8a5e8f59fa0b8c4ac8e7434b7d021ee9bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646849.7/warc/7f5869e8e0cd126f09263656936300b0e360c701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646944.14/warc/61906bc8df5a10dd7830a24d5167be91c05f9d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119646944.14/warc/bc0eb56684a4dfd6334add41a404d5f79c65273a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647626.5/warc/b46c47e4013f8c6b7a3a718130bd0f20998c4288.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647626.5/warc/dac281482ed36a6cfeceb7d96e9c6715842be96c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647629.9/warc/b3e5550566d79f4cbbefbc981aba2dc89b733734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647629.9/warc/d5771460950a57cd1f2a70810d9c66b29189a19b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647778.29/warc/677cc11942c864b1eea0e3506131b6bb3a7aa7c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647778.29/warc/8f2abee9e551fda58bd9422eef3ee001c9d2c71c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647778.29/warc/b2e04fe700ce82e2d88dc9875afd9df06e35df4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647784.27/warc/2345a2cf4325aeac5c7c84b94f697b7514cb1338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647784.27/warc/a4091a293915acd903c1dad2aa369c3c834ed2bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647865.10/warc/be6440b34701d6cee5f4b9d44e3d7fb8780d8c3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647865.10/warc/e3b87002b99d430c8c9cd50f06514743a860954b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647884.33/warc/1450b973d08ce7c9367c1be03f552337f19aa573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647884.33/warc/a126e1943ee506ba7507c2226318ec44e2dad0f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647914.3/warc/b719295546c9839de0e29ccdda8350f3b7b05a41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119647914.3/warc/cd89297a58c022b77bbf9600c7ff1fc745bd986c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648008.18/warc/4de1c77158b8ebad1491f19cb39681ccb9464daa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648008.18/warc/b00f388ce4a38d7d0cf8e0662e8f6fd4c57e5441.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648008.18/warc/f3c419c656247a3cd89bc4b904650e1c1b2ef954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648146.0/warc/5583cca218d0a8c3113a89d8ef65487ed1fac298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648146.0/warc/cb1ed0b80520792dbde759ba968820b7ceb2bb3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648146.28/warc/4729551694919fd17806bd7d59caac6c1b5ece99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648146.28/warc/b062e1c2783fa55f15890c26ed7b05310805fb47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648148.32/warc/187bfc0401332d92351df8e6dcf53feef9623b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648148.32/warc/d1a3d924652120b9377196ac8fa8064074fb7921.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648155.19/warc/2a20bd0c8ba0576e8a5b0c55c11f434ccc361dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648155.19/warc/d02eaf762ba3fcf12a591bb416899a9cc84e44c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648155.19/warc/f3578172105276c8f0dadc61c3cbf0d23beeb6d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648250.59/warc/7641975c6f4205891ff7640afd8b79a790154fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648250.59/warc/b04c230ff5f70d795ad01dae8564ed118e586585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648297.22/warc/6405d3091d355add8fbd9b361f14798c476399b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648297.22/warc/cf4d2ea9235fe26bf91a72919eb161685fc9b1dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648384.20/warc/74bfd0d4edfcbec034fd70ae1581355c9578af9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648384.20/warc/b49163359a35c85afe17d416e2baeeef52a12558.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648438.26/warc/24a3da9bd0429a52b381a986d7ba6a62ac2c746b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648438.26/warc/ee247591116e781c479634a46e627e4bd98f3001.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648438.26/warc/f92fd0f68469afd697fa712e475328df7953311b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648706.40/warc/3533be1ab5999db1e8a9e89a25f1c31d82b47945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648706.40/warc/ea81586e2868c9afb55bf6c68009ce4f956fa936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648891.34/warc/0ae71e1e7129498c613bc9926aa946353c555947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119648891.34/warc/5f9495b5412089d126ffa312fdf6522050ee54fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649048.35/warc/28c6217a137a4218cdcb6b242f44ca8760d7a750.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649048.35/warc/7def52daf9414f1784fe937f399a2265dc086e41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649133.8/warc/52d8b7c2aa9a0c06571fd7033b18f6ccd05ec360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649133.8/warc/e286d1ac3a322cdfacf41e5d46bad11c0c473205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649807.55/warc/4e608eb84964902f40f47b2b66164960eadec69e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649807.55/warc/ae2435a1478e6c448cca1c0571b81a4ec1df1f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119649807.55/warc/d80d9185c0a34cfa493d2947cfa414dbf8140e8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650193.38/warc/0b2671dc92470fb6e21b475ac6148e5aa8a548a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650193.38/warc/667c5e408aaccc78837e42e5353866daed4ec10e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650516.39/warc/744c925c0763658e9f449291f76a2189e6e72928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650516.39/warc/85525fee996b09db066c585696a0ef206125a5f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650692.11/warc/20cc462f819831335153e25c3e77a71c2480183e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119650692.11/warc/85efd81c19a801f8c66c2335190bf2995e52f7ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119651018.25/warc/2d2364b1e7f020ca461c310e2646e50cc45f9f02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119651018.25/warc/8490d3f743f2b60d1f9cfec33a21c9aee9688059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119651018.25/warc/e08140ea04cb101e463d62cd062677752bab5e5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119651455.41/warc/006a8ed10ebb370de750abe6a7b70b243e7354a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119651455.41/warc/719d0de569a0605432c2f36910d55cacc4127640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119652530.43/warc/320128d39f061d1c793b765f662f3350ba76e19f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119652530.43/warc/768faf077003d642d683b4af1d756d0c51179f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119653628.45/warc/7b6631a85b060ca158beda739e4df281af6b6dc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119653628.45/warc/84f40a5d0fd88fb6db9a866314df41d5e0fbf3ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119653672.23/warc/75e3593b4b3135ef24a8b1348378718550fd1f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119653672.23/warc/7f4b8acd5de224c629a11ed166857094eff403c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119653672.23/warc/a9cfb3fafdd85ac73671fe37e7c403b8bc1b8b37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119654194.47/warc/0c01423bf519742d1c785366d37c287b4ca646d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119654194.47/warc/2268340395fb12060fa50f6039fd315ceba5db73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119654194.47/warc/5b6267c470cb2d391c7cf3172fe0cbe1090476ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119654793.48/warc/32dde122ff4e50be7c4c4b625b7c358bb287ec68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119654793.48/warc/9e0ea063961e2eb52ccf00eaedad3f6218bb3fd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119655159.46/warc/56e863595be5c1ccadd6e66c726f047e75b3a47e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119655159.46/warc/64ed7d7a152c2d177d71f826cc3f7cb0fb80a8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119655893.49/warc/50cfb21b8fcb7ccbfa6b5c4ab9cf2cc5f2577388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119655893.49/warc/a5ebd32cd0cef967eaea24be2f3b607fb6e49793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119658026.51/warc/6fa552ea62c6fcfa2f8264225f6d20aee1637f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119658026.51/warc/8221dac36db27fc410650aab625f25bb18f2aaef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119658026.51/warc/855000862238146cba0ef457cde322242b3e2df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119658961.52/warc/6e190f9c29de5dda20d25d957e2df4ad7d31df6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119658961.52/warc/c0c6a97e5a987ab580f4d9e4a947628e697ae808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119660038.53/warc/547c2e6af44389dfe757bc3a3d72f6c52ee15f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119660038.53/warc/c1fddc671951135998731ec2a8f50a2a24c2f641.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119661285.56/warc/43154da8152a44ee65e3cb4f600311285fb81efd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119661285.56/warc/605251fd464ff26ee9afe48efd48e65a86e634b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119662145.58/warc/581dd51d9d7f7a3702a81e41a4e5c3b052fba91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119662145.58/warc/58324f24ff5a28d41193628a01cd356742f901dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414119662145.58/warc/eb626f929486b34a93a1ef99bdc0b10d6cfa1c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637897717.20/warc/d6ea2b674666e4e9a8d79672b5a639c09bc83c07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898119.0/warc/01e72f305a63f7b03d553eb9725d1c7e7cfd065f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898124.16/warc/2430030020a92d5a53843640f6f43d8576535e12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898124.16/warc/59a875df1ed4b7bd617547512791443f52038c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898141.14/warc/54b67a8b53994ba6eb77f40436463a89bb3a2660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898226.7/warc/1bf7d9fc86c258eaabc52fe30ab01d408bb61141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898226.7/warc/ab8b98d8d0a4ccf6b8d7e226178db9166731e216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898477.17/warc/43df6245760789dab7290b7688a6bfc707440f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898611.54/warc/120fa190ada6e650a99b53740e0d939e89492cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898611.54/warc/54abb368be1a9630495bb033f4046a598e14b9ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898629.32/warc/9ad37a9ed2aefd98bdcdc3e392c2bed7f3da29f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898644.9/warc/4bb49365d7aad38e8a95a2e9b759cd8ef7acfff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898644.9/warc/b77bafb2ad30ca6d39e8841164061754f101994f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898751.26/warc/c55bcd21b25902a3878f9697abd1b41d33f8a229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898779.5/warc/9816e763fb76efadb1234f60f64bdb0243047be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898779.5/warc/fce513d93178c1cf941b06b061a649dc2694cd6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898842.15/warc/9a00a1a85e1f8c5b46ac67cca31c33551f636c9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898844.3/warc/1e9de95996804868d44dca1bbcb177dab80ee589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898844.3/warc/5043e77c0af56469c7988637aa09bc1d623836f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898894.2/warc/0e276b3ccaf5a97c78fc675c83301568113380ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898978.59/warc/33e5a6ccf30677336814c73c6213212e931b0616.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898979.18/warc/9d86fdbde481c817d0a3658505f056b7fa783799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637898979.18/warc/a781fe1273d7f4876b2d3a0e387624c342cfd411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899041.10/warc/8bee17b9f459b10be941e551bbf29ac9d3b1c9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899124.21/warc/053d5c4f126d0978e5f7a20078ae9e1980757665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899124.21/warc/b44da2987d965c31c09a191f6ff42127f527a180.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899132.1/warc/14dad6e6ca638515d7d217c128b733d140cdda8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899290.52/warc/1114b483f3dd5bfc758953d06f42fd2075e54994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899290.52/warc/95618bb5a620c24adb841d96221eead9ed3c9f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899531.38/warc/2685fa21a40dd24594bdcd7ac402562188101b74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899548.41/warc/12f4a9cdbffda9316f2dd6d649ce240de9550043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899548.41/warc/4720fbab6ec3d114e10fd972359ce3afb8db3b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899632.42/warc/99aa4c96f94f4f8a067a0071d230bd8ef82294c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899633.25/warc/22ba06f008891fc77a37fb16c2fcf26b00a776ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899633.25/warc/842391ba74ceefc8a87923f1c383997e2fb96643.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899701.11/warc/b4e7302d23ae1277ef28ba6703597d5d5baad82c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899702.24/warc/7b844c0e07d5b786d1cab6f354eceaf33c238a82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637899702.24/warc/b926de28f2add50c6de352d951da8938f17838a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900019.55/warc/66ae0a7736249e718215f07de9e233dbd6c9c780.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900024.23/warc/f525e20669c33e5a3bcb8c1806c54564495bd357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900029.49/warc/5dfd6ee8c91b2bf4e9d19d60a7409f11954678dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900029.49/warc/f6e95c3f906087d1b0505e601b9036b7afbf3b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900030.8/warc/3acc416fc3b316f12873a40aca6d8ebab5896145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900031.50/warc/2033ce804558ae8b6e2759c1561ab18ba845870f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900031.50/warc/2666249e35635f864d51bc6ed6f3e51cac6a8ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900032.4/warc/fbd7986f2850cffe563f1560f5d425fecaca9399.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900080.19/warc/689ec910b95688e0906c9658829017bba7ae7f35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900080.19/warc/ade49aa900c80d5372cee2f03e74145e2f19e768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900160.30/warc/8159d58642a8c65d65d81e69c612f94bccdb95f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900185.53/warc/17532dc7de801ed85c991cedf5467fcda1332d97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900185.53/warc/46e811ee7ce8668c86fe15e12057fea654d25a1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900248.22/warc/59bf8cb4e5f54803f2263f413e857f9fc8ec5cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900379.37/warc/2cd0b166944d0ef6579876e80965b2f2fda9bcfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900379.37/warc/f16b04134122d39c352a58fd838955de54c7b137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900397.29/warc/82be68fc1fdfe67a9a0454439f8066076de63a89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900544.33/warc/a7821221ac209fd0194c08789f67d65f87a619d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900544.33/warc/b382273fb793facaac07f2539ce7f3b6d2a9c239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900551.27/warc/9f096b56337c85873808cf41581d579dda9320f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637900857.12/warc/e122ec617012ca00b61320b4638e53fa2d6c5776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637901687.31/warc/08ee6d7b995d02e33be1190e7fa2ba0cc5e32962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637901687.31/warc/aef7eb2baddd0bfe9b819c4db3acdb7f45498eb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637902725.6/warc/377031a9e02461782d37fd5db9188d6a67d3ad7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903052.34/warc/ce0ac16cc1e5fbd10b53874666a78d34a6f55bad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903052.34/warc/e0a9a495fe002831f6ff4fbe522fb5c65e1beb80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903439.28/warc/46993b20253c81d0909f56546ebedd40367ada8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903638.13/warc/73ca08de519fbcc03cc1f4204cecd67712adb238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903638.13/warc/94662f7a67606cd4ef09378970c4e487ac37ee5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637903893.35/warc/572baaf992a0a5fac8663199724fa7e95fa4d409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904367.36/warc/09df3d975605e38833240cbe59623f24ba8e59d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904367.36/warc/9ecb96df8685aa9440e779c5e7c601b6fab86373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904485.40/warc/1c7490d6a04c41763da69073a50636c1e24ab895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904722.39/warc/89ad7c87fc9db0b732d96e494d4b12d3cd7f8202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904722.39/warc/986830fd101649d40f8c4e8f5b7871c91a0c815e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904760.56/warc/32a77f1b7d9b1ab7df3618589fbb6e914140bb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904794.47/warc/1ac6366d34d858867b761eee9986dec784bd90e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904886.43/warc/47b09b062a53a5ea3b9c59a1e5f338cde7f94f3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637904886.43/warc/d6ae03292ffa7723009185474cbdd954cb85f1a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905189.48/warc/6bc41504a4d1068dc5b41c3e8e1fb047ccb55699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905590.45/warc/1a14175225644d38b2e700b4477bcd04372c4917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905590.45/warc/a08202ef1925b4af91948784521b44723ad45aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905639.57/warc/1eca08e2195c665aa0d35a5d00d975edf47affff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905803.46/warc/1f9b702ade7ed938308b8f3baf35afea232eaf24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905803.46/warc/455b99a26ee226a603c86299c6cdccca1ea9bffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637905860.51/warc/2189168f5144754fafc5e352749fdfd87f7599c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637906433.44/warc/691e85ee68384bb2f595bb4203ffe9298a7a78e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637906433.44/warc/9ee6bb4eb55e770212f54f4283bd88346ebf4482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637906909.58/warc/2ed8c2e1a1710cc89509edd3df4f66d4531ce92d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1414637906909.58/warc/c8d35715384a057031927b09d0e66fbd48cdd5ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/143c547676c11876e9ba54938a4924c265d72256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1467c4e36e3362feddeab7145e8128502cfc06e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1571d7ae11a0ee190b0b4bfcfd212bf84893e8fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/16c36eabcb0ff53af09f492368929bc36537ad9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/179b0ce92e98e07e147d1894810f40d62fa6ab38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/17db697ab4b93f52dea0432a084d80fe1f9c642d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1945b42b6e2c3ecc4a89cdeed9c20e4802f8c125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1a063d855f43f432f5f2fea45500ed2d072cfbc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1def653e160361a258183a3066559fbfce584f4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/1fa0436d32867971e88a93ed96d2e6c029dda87f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/2239575a27e962035438337f2462dd81770645f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/232cd6dafe10ecef61d80d31c519cc42539d1c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/256caba67a28b5e5e5cb9e1f1ac0feea33cc5402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/25749d64f878d02ee6e41eeaae2d7106ce7cde76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/27193df6dc9064b98d7d7a157f5465e5338461d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/2be307f00c851c43eb9642847bc4bedc17397a28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/2bf53006f5ff2b7f431e6c0925999da01cf1f078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/2c8551d0d37f7baf6e963f69fc5e50f14b4d9a34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/2d8a539a70727daeb2fdd56ecf02222ae2dbff5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3003c9b28e01abfee5c9a566fc33635493bf61f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/31294897f4b992a7ca619c1ce16977fc66a9c559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/32b465bf28dbb84df86bb841f15cf5f88b28f9b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/32f0b823bfb3d19ed6f5fbbf5125f9a3917ba03b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3399fb03e8e4d0034dd29bbfbaeaec7919660ab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3430505079b0130afd0d21418a8255d6a2016b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/343d9b0f63bc3f6b1f85a5c0d628071a8818505c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/36584f9cfb00be39e3718b61e70e42e30c8f1c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/398a4a2902712d48787bee8640617e8766e83b8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3b6ebb2906d35359a48b4244f6ca9b980bab7a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3d3ad6815f0ae38ff0696186afde49544441cdc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/3d745612d7fa69cda98bd9a1e6311092e51053c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/40036d2a996ab3635a3ebde850af481c4f305835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/405a232e8f0d73a111fd66363a59b725b9ea5998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/40e0839196a8296c64f9dc092c6c33c9a9ad1ff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/41b87ff5fa006ca96b57158b2c9f9de4501b1d25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4277f64a348a45f0efcced485f241ab8b87fd72b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/428156651b349044dc0bd76898a0b780770d99a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/45150e86b4353c32faf26223bfdfdc248c10d357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4964779aaa3430b24042605c34c58eb8d95ad803.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/49f2ba029d75e03cb3985cdc664f40ef29d18633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4bc997271d33cb350c0b9d66c5910a8f708c4d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4dc05cc879bd734769c733a0efa8c2969efe0f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4dceaee7196b3e87f19a95beb09c85dd97e8f021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4dd7583978ff834310c95d5709f05be150a13aba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/4ea041628c3bb94535366059f15bed4c8c0d0c57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/50cb8ff8bec612190d596eaf6083e0c6b1ef6d76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/50d49390efe72fb630b7bf8c9d6dbea3923acfac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/522e48638f1ad7d25c4b84adf9c8e50be36be8ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/52b9c25b289102aa3a19640b821cedb9aaf13b68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/53151acfe54fba5ef728728727960990cf1c8c00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/53ab6772e277f418b5a914902174238ca401b10f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/544223e6aec4d7827851f4d2d3918d781c74c1ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/55f027be64de1a81ba871141031a2f77914bbe49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/56d819832bd417649393dd54099e5a0dea0288ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/56ebc6df48db016dcc0298b9080a6aba3d42f69c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/580c75f28f0ab9cb79112545a94e523ebf6e3037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/59ed205198155dc9bae7c33cbc180c346e6a1463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5b8ea00de1ecc2f3fc92c716e80ee6aeb7943836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5ca6022c9a29b3da0c151ffbb095c39616072d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5cf6efc3ca3694183911ad902db0adf0333b3957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5d3510a0768cb7e44ec7f347a41f9fbb55ffed09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5d7d5995a9e2691774de97ebf49b02f99bf59453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5ea9dfec694b589d4b1b01ac5cddce3846a59081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/5f521832af0d67893b8fc554d55f1426ca52c96d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/60c0cded91f77ccec5dea35d272cbca7eec9624b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/633836f1448825c4ade465e239db4abcc0e9ad33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/64cafcbabb5bce02960033fce730c99d90523839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/654a4870ef76625e1876e515d03b053fcb35deba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/66f825cace453d186d3bbc65939a67481984bc7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/674aff72ab8abbe3067c806253ad45215bef06e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/689174ac57e1a2e36dfe9d99f0a5c258de5833b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6a882f4d48c8537b7c490c6fc3c0d7d6bfbe819e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6b39909d87228360cf5fbd607dde3b4d33a56467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6ca5dae555cf440690e90834f97431d6de11d3a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6d2c363d32229235d8d4133b1ef8f2c9733b567e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6d3593beba39bda61a7eecc4270a2f6714c08594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6ec817649cd09ba9ea181d37aa459906cc8ceb41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/6efba7cf4804a86ae150c1a4798dd17ffc2d69dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/705170a0a7bddc69e58b7904cfa3dd319c2e03fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/712954404265551e1497bf4ca9ff326b38e7d3b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7207772ed49e4cb7c8242790d9a06182007230af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/73bf28aad58f930a8136c99b2cdae46662bdc1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/747389d0b4d1f0aac165542fb4332a96eeb0dd59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/757914c314afd24386d7a32cda845f55439a4033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/761ea274f0ad23e7794907736c61579a0a3799ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/76da536291f6d3c6c000658ffd1b72290c0ba74b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/770ebd93aab19708f6d6b9220ff8bf1026ef254c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/79004b022903021af8ade2e6df3eb7b716f36f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7b6b83479581f9d60636d96b3265fb3682b88f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7d1cdb44795311b02467f2e585548c2c7bda9889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7de9cb7c1a901f707f713d7b10015f88e201b276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7df267eaf17983ebcda299091e7e26eee0fbe6e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7e26d711d6f0220870a0cfde67bcc7ff408d7545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7e5cd12ea4b99cfbe198ea8a5c890448caed94f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7eb0758fbff26372aeb8d3740d2dc2e386ef85ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/7f66be7f90a31ea6fad201c21874d9349bdc4214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/804354a52279b30e205f5acaaa3a043d9768d1be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/81aabc844aa78f6344d14d1062be44e591ec2f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/83e9d54b72b878899accbecb35cc1d8bf2417eb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/84371e8919086e83878bdbab85be74245b7eea33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/845ee40a0cc9a03b9f6f5fce3a09abd032562ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/84a18aa9a5e7911c0412eec03add4b8550b1743c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/894784240c944cca29b6428398fb12eaf34fdd32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/8c4e9eda0af12e5e665e13c78fddf342d328738f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/8daa9d50b35f123c944529bb8ec993b5e115856a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/8dc1eec4eb8a4b235ee5878ee25fd7573a54d758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/8f6dea56332ca33a2783a1b64acb21728b090e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/904aff1fa02e73c39db584275ae537b68b5105dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/91e45152dcd9b8e6f21c6ac9b5d88b6b4f38e1f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9263ab71a12ddf4a77ba3679f183b2a2ad23e501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/937c7e7e8970e42b8be2bb1ebe15a25b07a0ec48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9418ddc69a89a0f51abee138888e7b0f46359c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/96832ce9e66131c0b88d35e1f0bbe3ebf75d68b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/99c703f01f39ea72d3a1c73010e731f3b06ea725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/99e6b3a85154fc1350efac0fcc79f3f1b490f43b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9a08d191ea53e8d682b7be8f60544153f07af0df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9a9fde2ce9aaa0710a12a70d4108be0d74bd5ae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9b38d4a2354e82e9dc7688a29f4172b2e94f7061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9cd928afea07b4ac6b080a74fb670f74da94c667.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9e1acf327c3088fb4c5a591ee9b13f4eb76bf074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9ef678d1621a9a15b95dd7b825bb847974f05966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9f3246846744dadaf01b79c92a087a5a6f971d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/9fb035252c5760c5e412638b607ef5a806c5c53a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/a0d6dbc11dad202138ca2677214f91c2b1a5e4c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/a33a4afb2414315a3e841252ab0324e8e00f9c13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/a3a167836a70ce281d1e9902b9109042f241c34c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/a422fa640f8b02e8ba7a718e44884088cc1874b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/a987ca0a80d7e10ec2660c93fa8bc205ccda5f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/aa0fb49bb36b8a97cf49460d1b21f85ffad077c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/abd4916aed7c5fd29bc937362a23ad2078ac1b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/abe2a87b5b78670da291dff7be6ed2885c829a0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/ad6b995f37d9f1ce4c7a8ab61dddf0112431c41e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/b07aafbd88e6df3a3d5fffb1a622c99bebe4e7b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/b176081630023bfc7a068483d32483c368808c53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/b2ca29d2a2c3de29486c1ecfa711814e6330f57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/b8c76d6acad437f0dfd1a1d933281445efa7a7fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c02f2ddf23662a71652436b8ff15bfaadd6b9717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c0b1335e2767e6689d02e21f92558b9fddb6f7c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c0c0de969097245bd1d1a65a00f3f34448ace69e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c0e0fb78030ab3db52ff5505fbcdffa273be4b30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c14ce17c6ad2b44f72db58211d611b2d3559ab62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c4c2fb2fbfe13ff268b47f6110a850fa5f362bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c5089af8988f5b76b2cc2cfdcc25ef991177aa5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c5d980be0a2b81fe2e028adde07a728fe2fa57c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c98d0096ee016880e63897fa4ee8d85fa77d151a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/c99553475d5304ce0925d36e82c6886533333c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d08c08df543ea3775265136b91503869f10c4c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d4e36facd0a5f68314a552f981742e9c527d49ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d5ce95cf720ee24fc3d9a7f20e721ff47879ab10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d5ee94629fd619092b3f083f78186702acb865a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d666f41523d8c11a42f45a82c87867ac31008ff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/d782d7580cd0b780694dc8f2a294fcce9a0642a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/db124a28d6105e8a04a84f459deb1e7e6bce8531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/dd6bdd61be08f33ae13c58e0a3cf93a6e6bc76a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/debcc13b4a43f0b9c6097134efb906b398473f55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/df5aeedb00d68ecfe84536b5e3563db8e6411c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/df5d1e4f176ca48c31e641324cc7bbbef6a9a06d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e13b67fe4673aea77a173e73b143aee67143c0d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e1b291d6d17070f9d86eb055a149a15517e1801b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e29cbe9c1741082cef94500ca8f1df16f1234a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e2c3785c398f2eb6c9624517d272f5b2257132e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e3306d45907c40304e05ae6cf0740701aa101f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e594375b062ad5edbbf32b3de0fed4275e825d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e6e31a3f25b058f4180689b24c174aac56da4a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e808da4a4a1049a25c34f4cd9564972a3278f436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/e88b9d551b485bdbe9ab07a80c7d47735a024d0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/eb1ecbeea2ab846fe569400ee523a4083f49d2b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f0d1c6c460bebed343b8d94f1fb5c5d1673a1e08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f108f6719fbf072863c2a7c6652c46510544c6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f28ae83122baca9eef7d99afe59246144c3fb6e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f4073d7eb5a12b03105c03efdfc1b1c42e4152aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f5a16e6672122b3797acbc9b522c03b190eff4a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/f6a6e31709b442217f8c4e50a3e64e70d04f4c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/fab6eac8d3b669b217108ee44ee83ab53c99e964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/fde35b3b3c6f91ca4e05f848ac9844a7a8423bc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/ff6c1b5d90ba7573d06cd778cfe851ff2d56211c.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-42 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-42/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-42/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2014-49.yaml b/configs/cccc/mixer/CC-MAIN-2014-49.yaml new file mode 100644 index 00000000..8ec26159 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2014-49.yaml @@ -0,0 +1,429 @@ + +streams: + - name: cccc-CC-MAIN-2014-49 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/00d1041c111683b03f474483a57b1e0ecac6ddac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/04bbf7e8e2b64f38d83631da22ebd77d034f597e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/053230a0592752e5f22f459cceab46701bf4b593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/05bfeca10e4e064d5df81cb6ce2c810e3e350197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/06f3717d048bff556452982504f1b8b722264087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/06fd7bbee474bc934b18b39dce44b90bccba9c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/09458f9bca385dcaee996717f2fd83ba06ce2cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/0aa7e1f44a4b69a2a129806f0a4b657ef3dc44f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/0f426f391f9a56cc853f0d697e3cd0dc38358c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/10a6998744dbbf83bd3514ed0ab45960a2fa21d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/12de672f28282104d28dcb5db2fed0bbfc9f73bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400372202.67/warc/c305e2b4c84006f81c986289ca3316cc57dacb8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400372542.20/warc/3211bcfb7d2076fdd5653306431fe706c10a6af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400372542.20/warc/3aac05f21512c5eb2a6fe39e7b85e8610fc2a95a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400372999.9/warc/14f1b1ad40900d3c3bf1fe701d93945fdc287a0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400372999.9/warc/cc624f139d57679ac6f8712b2b6df0f3b606e5fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400373050.63/warc/cfc7980ee40cd59b6149d540a26d679aa12399f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400373301.1/warc/e18ec0f7906ded18eae9a577aa100531785ba97f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400373486.14/warc/8c225140e89e84a1c20153669bbff80d64d60ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400373486.14/warc/a2f658442645a2a598e2171cb82b833fbf9a89d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400374040.48/warc/921cbfcc1167d494ae1221596663106d905add9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400375630.34/warc/ca51292fbcb3c7d7b6bba6b6d5c095c104ed0cae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400376197.4/warc/1f5b25c9c18cbce6f42828cfa0d4ed0e39c0fae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400376197.4/warc/c3b4dac7e20e29062f6944f3bb8c6fb64e868dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400376728.38/warc/5527d869db8ca5a8da4c0320fb7fc495a6763db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400377225.6/warc/fe116f6c85f1da8443f0e99cd8e33fd46ed1d0ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378232.65/warc/37950731b60dee293e033dd240fbed0a40d30d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378429.52/warc/c20d776c9d8b8e44c8752657b3838871a038d313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378446.58/warc/490f0ad331343ea41cb9c8561c6265e260b54f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378724.10/warc/64d104eab5ce5aa2b988739e51445484a5b12591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378815.18/warc/3f81288a2267ea8d234a4e6fa21970c3d024d73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378815.18/warc/5e1794842f5172187bcac91477481a4dda25fc11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378862.11/warc/7653e6740e73c88b1c070a4746bec6cf2e6bd5c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378862.11/warc/94f4070eadfad20a1fcfc7876027a73334ca47cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400378956.26/warc/f44ed5fb64be2e61f78a1522710bae139feea9e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379063.15/warc/12eea097e4430d456878962458c5e889e0a5adc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379063.15/warc/ab1aa647c1e2e69daf4b4d163a957e124e3b9ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379083.43/warc/a9bb309b5446d74bbebe26d2a22e3c797b63c1da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379190.66/warc/3f316c700b213a4dd42fcdd09b86c651f1d724e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379190.66/warc/abc65c36834ea0fac13486f3c1df196564afce69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379320.39/warc/846e3536929aaae5c77a5739c615f7e7784c6b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379320.39/warc/9497925bdb14f3242106e39c3c43e83bc643bcae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379355.46/warc/6f858ce2f968103c0e71a46df49a72533d0d0343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379404.25/warc/1afe2d1c00114f5d83fc76c3ffebbe8ab59fe857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379404.25/warc/5903f74cb1b702ca5689f0973beebdec6a207da8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379414.61/warc/3dcdfe6d47dbdeda9bd5d3d8ca56aaf286fa9d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379462.60/warc/b921c9d70ff564184a38a6348ee70148c93cadaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379462.60/warc/bd18fed2c617f5dc53185af6ff2f19f520479304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379466.35/warc/10f388b9ac13b856b9305c528f3e763be71db599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379466.35/warc/7f63e6cdb322c856dc510263b4e877adea47bbce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379512.32/warc/d8a11ca66ccc5d0c602005c638fe2777b2b2e619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379520.45/warc/2572163608d1570bc99ef6cb027baf072969999b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379520.45/warc/34483f1b55e5ad54d7b91feec738dd68373c6961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379546.70/warc/509c5f0650be372e161076f65fa383fbe2391573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379555.49/warc/1aa27de8111498e9de66b453983ebc789be42372.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379555.49/warc/52adffcd91c051f84912112d7e32aa5fc6a57597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379636.59/warc/008f2a2c5c93d0506f049037816d09dd78e05585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379636.59/warc/f4a8cfb78a3dc924d95a9111cec4c1fb0af693f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400379916.51/warc/a52d8c07a8afc902b180732e380c257dab980f53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380037.17/warc/3057c716754f9236e756f3e661075f55209f9259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380037.17/warc/6972e7873efda19c37263db262e56072b9cb6b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380068.47/warc/d454cbe24b0a9854a00b32e8d566b24e9cf17c3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380233.64/warc/48c434299382cb28d46ccaf95a87d1e614f02cfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380233.64/warc/58608fc53acc270c57548fd4c8aaa70cd3f0741a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380236.36/warc/4e93e7c98b30660a011f3bac9d3235b8b6243fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380236.36/warc/90bb3d34f5f75df6afeea424a37a2d00b46e7f02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380355.69/warc/32c28f144754332b4ec6388ff2dc8bbc9bb4fda9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380358.68/warc/c4ed2236df06342140771c91bacc0607e4e80517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380358.68/warc/dd474d78cf45717b80f35c9ffafda9c8fa263cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380368.73/warc/fcfcd5f5c0014f45c2c655a7f63ace4207bbd062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380394.54/warc/278f8bfb5d38f375ad0783322b571be6d27afd4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380394.54/warc/bf7e1b1d2b2d0582a8c7c81766958ce0770c00ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380409.19/warc/460d9363be407429788aafb15b36aa4bd3c1580f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380409.19/warc/4c3ee407380014cef2a143d5943551589ba05941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380464.40/warc/990ba543c38579facfe374428f38205a6811b233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380574.41/warc/8d1e017a50b322cce5ff2b292ea6f3dba413aa5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380574.41/warc/a7c04a253a64c8f5a7e1c619a76dfb0e2cb34174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380627.44/warc/8841729417334fd825098c54cf76777c0993c791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380638.30/warc/4d27ed66868f24870d9fac28fbd1140d9b01de96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380638.30/warc/f420dc5ea94561f101653de5170f1bc045fb0585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380683.31/warc/952af9fa8592f9d87552bc6297b0bb5dd1195fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380683.31/warc/f50860143e0961846ccfd3b5b7b449940dbb3bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380866.29/warc/ed7ad586b6b2159d6b404d3d81dac99c1b0ebb63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380948.74/warc/1b12f164c231472a1544ae6b4273b7ab2153ec3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400380948.74/warc/ab6668de5677d06d6427abc9d09c4b4a70cb983c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400381177.56/warc/7c846ee5c267c93b3c8260dfa8f5c33c07db7592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400382386.21/warc/152567a5550eae7a02dcfa9e5cc8bca192a47e0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416400382386.21/warc/3c6e1cf9f63cd6a3c33cf3f84eec90b31f216ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416405292026.28/warc/bf68a7ab2fe5c2dbceb1f3902bf72dbde3b81290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416405292026.28/warc/f8cd1ee169930d20d49919a845c77a97e6005832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416405325961.18/warc/a0b9ab3f2c36e47df94c272707b14e3c4fd73f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416405337312.65/warc/a84beb1a23c3f41dc41cc0acf25b202c7217283e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416405337312.65/warc/e255fcc621a3888ae777ebe99c952216622d5ef5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931003959.7/warc/c6b216ec64cdaa6abcbabda2c10f5bf09adcc7ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004237.15/warc/05867b5a8af94cea9578b0ea88bd4e332447812b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004237.15/warc/fef6ef10e80e255aa63ea0761d4c9c821f90f3b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004246.54/warc/39e1df85ee6e616a6fdd5b107c0b9ccd168dd088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004885.2/warc/381dfad60640015a1f0abfaa95fde58abd9db1a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004885.2/warc/fa9de6bb81ccfb040b022d1f61bafdc702d74dba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931004988.25/warc/82860a0d9c60044bdeef1b05bcea27a4b4c4f6d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931005028.11/warc/a0c0e1337d7c4ed59097096f087bdb9f31a6db4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931005028.11/warc/b402ad64a4568a63ea5545db5f7e54a4b76ac865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931005387.19/warc/c1eabd4339d77a637079a30d5df1fab882fdabc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931005799.21/warc/46b525d14139a67841f16ec4658c361f9cb7dbad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931005799.21/warc/7f522d19fc665f8f478979ba4e448c6cbdc7117e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006064.45/warc/9ad6bb5b8a806d80a84cac9b8ed4c0b8164a776f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006593.41/warc/d0230c314e509f62d3a226858db26095c873345e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006593.41/warc/e8e6257d76e46bfee3af1f06a03d2704f727dfe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006637.79/warc/6305f8d59f7b874fe4b94b2bae11cfc993ca659b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006637.79/warc/7f103bea00a21a517094c0b5807fcd40c7c148c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006716.69/warc/aa96d1cf25b0fdee6aa44800417e35b236e5c9ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006716.69/warc/af07077fad7eaa6aed2dbc156bf800b9f844d7be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006747.51/warc/37f8d3ec5c2113074a87623be29012096d1cb11e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006747.51/warc/b5c85adae7232134552f9246f545a649f9abe6cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006855.76/warc/dd2db65cae4468527f3f8f8ea70e66fbc4025644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006885.62/warc/174a61a416a6b3cc031e55375d71c814cc12440f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931006885.62/warc/5aec2483e24a468f0c5eed3175d9440e0767ea39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007056.0/warc/3c62be5d8c903f39f9d4b47174f5577a70ff1105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007150.95/warc/58444b271d55261915a7b222dbcc6a24ad981fcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007150.95/warc/6ee9dc5bda6a54641d418e2ad32695f2ae7e8a96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007301.29/warc/720d75419d40161a39510cf7899dec7352f00791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007324.75/warc/06ec412a441127b6f6ddc65c8d65c574dbaad016.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007324.75/warc/18d0a07827cac690346f27b2f356c70180ad2b14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007501.20/warc/45edda8c0911b6e61cedd84ea3b1443ad25421c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007510.17/warc/017b9809620b95e4d93b64105aee84e193435f3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007510.17/warc/7873844357805a8fc1da514bd16869535e35aa9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007607.9/warc/baf560a49773b9152b61a864aee3beac067b3786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007625.83/warc/0615314a4fa1f7f0a6c30c9aad05f27da0aa4298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007625.83/warc/76f24202aa1697af63e6e65ede4a1e82780df07f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007715.70/warc/a23957e6b154d270f465de8d227af4b47241472b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007720.74/warc/1882e8590f49be373f21f8d92404e1a5ee11fba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007720.74/warc/dee218c401442ba397e9d416e4fbb8fdecfe8e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007797.72/warc/eafbd03e1434815906ca10e74adf8b131c0432c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007832.1/warc/674248a96a402b7f54445c72fa930b9a38917864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931007832.1/warc/fe23e12f07cfa870be29a90dc913155c6c9c25c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008105.47/warc/35c69104063da7d467979b01ab8c841865766041.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008215.58/warc/4b89bf50de0869363805a948372e6a5e060474b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008215.58/warc/555ea0a3c1488e77521aecd6b240ae1aa5ebf514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008218.28/warc/545e2cae5982174878d4e9323e41c6e761d20829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008227.98/warc/363e0b519cfce1c2b0bbfdd3d87edb9a3c6e67b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008227.98/warc/aef9e08e31fb86e78697ffe54acaa85ab4852e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008289.40/warc/88ea4fece8ba8e54772a884afa5806bb76b24db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008520.8/warc/11ee39a59ecde4f3244112217db4d7414d3132de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008520.8/warc/ff68aa22804bc51897ca56d4a8843c6b33a0adbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008720.43/warc/e67a9e8b91b4600df2aee035a835049626f2a732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008899.16/warc/191e5f58e464fdbb7adcddfdf3bd9df640ce7f37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008899.16/warc/3974ff5deaca7fc5bbc340b8601a00a7da850119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008919.73/warc/3dc80b23116a3bebc719f09df0950b900f479e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931008919.73/warc/979e3d0ab19c8ee233a388384d362280df0ded57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009004.88/warc/5f916f290a1625a85edf38b322b68626face0237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009004.88/warc/749bdab7b42baa7f78bc819112b7eb8628df21bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009084.22/warc/c5d24e26a17db55b345337cc720e44a933fc4184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009084.22/warc/e256399d7b1b3d91c63d769074ad86e91ab13eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009179.34/warc/03ca78e82985ddf567c4664164a0dcb74ef13eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009271.61/warc/b6e22ddab2b79e2675e9225676de8ec0fee7ba38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009271.61/warc/cec808dba294eca6b18c48d8266bdb30b1144ccd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009292.37/warc/c21b1fe279291c6becf4ec508fdab6194de095fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009295.92/warc/8069b9ae2060a9fa10cd23c2fc0eb116e9d23bc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009295.92/warc/c11373867e7f906dc9c1cbfadc26c7b513bd37c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009515.14/warc/e96b6bbf3979d5a793eb81ad1bf03f80213f0154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009551.23/warc/c527d5da2b125763d2d3624e14f2e9e9741d3239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009551.23/warc/d7440c780a09961424ba3cde4492d5bbcd0d8404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009751.93/warc/889c730a270687105710a460b8053ef14fb3798e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009777.87/warc/72774124114306823d4dc06ecaef06915fcc97b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009777.87/warc/e80de80b982c7505fd48b89d23931c6db77b1bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009825.77/warc/88b372e2b13d07e1e0d8c9edadcaebfdac95b393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009900.4/warc/521e5ad728552838095eaaf7cc2c9b03e1a7daa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009900.4/warc/7a2240594876ff143588bc696ee7d15d678c094a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931009968.66/warc/f9c1a76c7c0fc01ba656a8b32837112144ee6b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010149.53/warc/14deb4dff11ae5c9649eb91e1540bac7597b77ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010149.53/warc/2f0cf5d2ddf8738fc1907a2fdba03b495c0cb4ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010166.36/warc/2917a51087930bb52ee7f3c9b5d963e79bf65486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010402.68/warc/4ee4bdd0fcb12fd16f2330cade17473d92279586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010402.68/warc/c3a81a1fcc10c7ad659a9636f9aef232b41103d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010469.50/warc/9cb52384bbd4346caaaed7678b33762af99eb812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010590.31/warc/0052e6d2e80d85c65e71e360921c019e81e62f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010590.31/warc/fa7f82994eb581c59c4457aa7667a90b1e17c426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010631.60/warc/2f1ed021611db8971395c01ee701148971bfa5c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010713.38/warc/323ede49104d2c641c85cbac22417b0ae32278af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010713.38/warc/c5ce598d01d94556a823ad837c15e7460681568e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010792.55/warc/00fa0f8571c64d6027485635e38aa739feb819f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010938.65/warc/57dbebce6d4921f0e0c9f9f3d1fbc93a4976d181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931010938.65/warc/dd02d371205eeb76e66d5e8ec53894951b0c7e5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011030.89/warc/69975b9adfcfd536cc7e2087779681c923eb5f64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011032.12/warc/69872aaa7cf3324d8ff529faa69055cd3616a5d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011032.12/warc/7f5b7768b99c95a72ad176e57906b2a507e300a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011060.35/warc/7f2c1f85165bd93401013319e00bb13683f3a666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011060.35/warc/e54625b4099f740d3cdd4c1d7a975cdd1e1cd24d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011456.52/warc/52f9f34b0ec922cd41ec5f47bc23553b17aa6212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011456.52/warc/5619475e283c24edecab42e030f813485d971458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011477.80/warc/62fa8a8ae10c533b63bf44f61c8a2ead14f52e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931011477.80/warc/65bab9fc26e29469686f20b03f28567018a06226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931012025.85/warc/f4d91aa513de66ab846e06eb734082986e6f7989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931012703.33/warc/0db9cd5e0b0076f2302d25576d0e76a38a288fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931012703.33/warc/94db28bedd7db43c507790de181e075f90097f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931013466.18/warc/b5747209fbd24967a34ffe7e90a32389a2d9f39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014049.81/warc/ca50aa4a7385f171d29916e5a64f12b040cd4ca6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014049.81/warc/fcffe255c0c540faf87dd84ba0897db9bdf15a71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014329.94/warc/fca3bfe8994d7b6cf6b15fee7aa41e0beacd6119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014369.10/warc/5aaa175fa8fb2bf174a89e49fa3f1591fe6cd4d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014369.10/warc/a127fd688c4cadcb5e78e4e36818669bb4e08f13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1416931014369.10/warc/b0f9dee89d8c5209793f0493efe9b82a5e22b5d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/14892c81e04bcd0dbe030e7a1f956f3a30c19e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/15718430c694b8e05f4215edde7fba8102da4781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1601135a9207a713bf749a51f18659b7b56cdeee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/179239711b3c10b97159df920d1aa671927804e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/179f18ba46e4183a54825071f7e3b082bd7652ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/190cc5863ee51cb9507b4fcf25312522c86ced92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1d84f17adb725775e770ca7729128ee1522144cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1e98f26c2d46ed0fea055f515598aa79354ba193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1eba22f7f0f8a071e35458293d9b4340abae4d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/1fdd16dafe471f8a3cb38c49b7df8b14e9ca85bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/2423800935ebe2f15831f2d7c3ab2967646a3bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/247201208ab703dd8c263f3c6791f63971de25f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/290fbe5a3163192a88ef121701f812f5c7695fd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/2a29bd4f3b5dcd0323c91f471edb7e5be75b751e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/2a438e4554e172d12e5ec00c41d5436660c9b97c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/2adaa713859ee810152c1f40663395e6aabd151d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/2ae2352b4191ebfe58dfc5c1331944f0463425fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/32a6d156278ff22e18c0defc90bc0a0faa250a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/33398c29e2c2c3ea7c6eb049a49bb32a1469530d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/337667eefec9bc05e64fb331388deca2389f3d98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/371970b515e0149b47f66fa5f562ae79ebdda5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/395ccbb6fe796a7b052860340e04bb5ad06e243a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/3a2f5ac46c1591ae51f57e6250eda03269d24b66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/3d44b8777d21b7fe653ba66dd248e774a2e7f190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/404a479e5c53e9263be133ef8bb2675ca0ec0de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/40d5b798166d0270ec1cac6df4bfa6780e31e3ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/43175af8fb1822e292c1299c671154656381aca7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/4445943bb01fe1b768b231f0d28a865795df90cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/44f41cff78f5fa45b3ee5738a59ace44dc1f7102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/4603c70588ec82dadb8692595c5adf4f5e449d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/488e13cf587f0dcc28dad88a66bcab6ae72b1d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/4fc3e3d41b6eaeb2a25afeac021a78ff5076c90d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/54287120e0e2830d9c185f5654f7aabaf7cae7fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/547a1faab531bbef9b5ddc208086613e34a2dda3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/55aa3124fcef410b2f6dc29c416f8ebb26eac221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/56deb5bb768dea7dda83dd42a843fa9a9b6b0692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/5cff44341bbd9b7b6d1e9e92d21053747c221b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6014431fef4a95785c284a2741506b21ebccd4a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/60726f59bcb13b0bb0245e38a1e972430ebc14d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/615c330dc16b539e7b111b33c4b3ae2e21f11c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6468664adba7f7b5f50302ffa054c0c1259e5e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/653e420ff9b1a28c2a338ad2c4f3ac1d78c756cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/656da1c1032555f42f0af29684c5926a49548a7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/65c83e5f1a07d42b34949ee8e5b3299a3ae30f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/68dd337111d790f124acba4dda3b72829bcfaf92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6a26ee41fab23f1a2cf8d0cfec244e1cd2f62f3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6d9340939e5ca743ae91e8d592f94b2df9fca249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6e5e033cc5b07f51e5f91b942b8e71d0b1d9cd41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/6fca9c1fa6ee41af30087daae28c8286cd43912a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/71cfc310385ee08e7eb952ee834516b21308d686.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/76065a9f163a7b9c924edfc19f365840c1718eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/760b6f14ff734d9d795d2b023dee4fcee7b53c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/7a166b2452274520c1d371b27ca778db7859bf9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/7a6da8b9170d20093701e6572acaa651b07c159d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/7bcad6d9694c9fefac8f73d8ad440bcdbe81c796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/7bfb9700c20babc371ea0f867b0b5bc9fd3b47d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/82086df3c8fa6ef807a6161b4426ceddefce95e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/837fbb08e846a3ebb3f0a3ef31ab9dcb079d5966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/8fc140b97388afa5122dd2f5fe334f96884cfcde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/90af538ffea1b4919617aa25e08ccae77d466496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/96ed13cbafd82f5da2ffdf407d6ed370caba64ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/988b7b9acefa39e10185a35e1fdcfca0ec6c5987.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/998c81c24fd36e84312d9c28cef75eb4316b81d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/9b6b784e561993c0781eca1ed2f36b91d6f261a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/9e802da78ec4fd8a6e31a19795b8297cc37cc269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/9f8804adbace7d48496c74fd7cf753fea2538eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/a1d61317ced38796997c0dca5d05c9f529fb44a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/a472bf16c6fb6494aa284a9732d354fc0f82a5db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/a61c19e0db61a613c0e925c82fd8b546e177e4f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/a65cc5d9281d8ae0a346b7204447200a70883349.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/acf2a6053dcef2a55a48f312171141eb40a726c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/b3a6282c36bcf93d009195936ec2528155d4cd45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/b5aa1fb29ae6b11ad0ec21d714bcebdbdad9643e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/b912cdd323c410ff8bc4971cc2c832815779954a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/b99b97c497a33d2e2c34a0ff109f917f6c5b68f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/b9d7ffc7611b4850e11d496b15fc29cb40b1c2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/be189230661f05e5af73960ef1b581212507821f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c183780400c5c17dab008b80034a621233fc3629.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c408443ea7db3973a2cacaa1721ccd9e96abe53d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c58e71da4d806688fc2dd1c473041f28fee2d7a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c5de6c8375cd4df9993751b1c9114e788199113e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c74235ef807e5a9f79ad5aeb93f6f02e910b9d21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/c95a3e5e0561a9bdbabe4ea51134d8652a4c07bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/ccae7ae9b6f464628d20728e0df796a310aed863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/cccd1b04971eb766098f9599430c4bd75c49d9d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/d04d523a8eaef6a4a1da208ef81ec04e475f3bba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/d142234dd527e945a568a3b029338460e2e7611e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/d7b1019c70d34057b309e42baa3282bb83769736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/d93abf1dceca9abcefccb73a31de187589b099db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dad33a74b254dcf61d2ac93ee592c48f5d78813d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/db136a70f64c24ebda392f0a2f13cb7df320a6f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dbb7de9690f39b67e85f48186435ea28cd916e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dbcde0314ce847c0a7bd9268892edba79a82efa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dc714bf60fd5a030f84ab17dcadeb0a1c4f69a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dc8df94bc1dc94a92fe9cd9faf4ad9fc03ed09fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/dd60ff6d85c37d0bf8e33d30eeb4b81d4bf85899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e0365870d4d91c350c4efa664c5d2a08ab198004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e280fbb8914c62e91ca78e085a17a02eb28b4666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e348b0bb3b0b74fe81554f3a7c8a84b0ef98a2c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e3aa34340d6f43c0f70b51efb97044d4afa7fd88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e4995f7d4f388c03bc15cb1a8c0c03656734acd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e4ac4de55052add75cad9883b5851bb1b8dd4a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e7b1bd19fd8cd5de3ea2d0eaee1d3b2ec9dd6990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e8401d1bd64ff44a0c54e7d9da499fc0a246dd68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/e93e017f09d21d68edb30856ee9964be51d3d0ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/eb563cebd3c3750dee44404e83473a30c99dd519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/eebeae1fe0880503232342142320804449248dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f453ec44dacbfe2d7b5a92fa842b6501d658862b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f499d9dcf24030a746ddaa0a697cbdebd7fb5dba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f521162773d6185184cc7c1e3dade5f4433cd1a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f68be817cbe7b48b6553274bf8236dad9a006b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f6c20744c950836653564cff52fd3048c2a5ab2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f6cba2f083902258de4105ee225e7f65503afe93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f7fa9753d5da52ccb7e5eb60abcd7ac5bd456b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/f91ad474834d0c8c600863560fe462a45dd08ee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/fcae552e41d956e45d167653f67e633764ebdd4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/ffeec43fa9a5a69caf6bb1060c0eb5c659229211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/fff754a56b4a2a6c1e7a6fb85c5530ca0bc94d19.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-49 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-49/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-49/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2014-52.yaml b/configs/cccc/mixer/CC-MAIN-2014-52.yaml new file mode 100644 index 00000000..51463440 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2014-52.yaml @@ -0,0 +1,546 @@ + +streams: + - name: cccc-CC-MAIN-2014-52 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/005d1efc732719a9bb375c5e8d786e36583ca8df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/00f9a06955248979664af8635bf28aed90d74504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/01a038061fe8847e4e12ff1d347c01811bede443.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/02fb8a7e2155db129dd8ea7b81c505cacaf2eeba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/03535ea6b9c0b16c32f296fd5ade492c2698e5e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/04b7b6aef20a389b9f4aed7a2bd5ac438418b50f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0599b28dd5803715a29c5a68dcfe17e2455328ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/05a4de914219cdf9a2cbe181b0fc2d38f83980e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/063bac7f20fac0ebd9a4f99258e00aef2ebeb1c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/06464af511eba1db87d8fc8eab122e3f85e79a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/069dd0b3b77f9d84afb3fcd732cde24b8bb3bb53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/06ba24100e9b881cf64afbe4934100f0a26b2f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/074d5f4a155e9a3611b354f46df3f6492bcb1c0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/07f0ec5454a15cf643548c0f21e1a6bf8aed9a4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0850859d554a46fcc95c4d3ef07282af348567ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/095a915fb84e410d1cd8e158d496edd86e5b66b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0b52b48f76462370d51f21f36f2007d843c29929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0b7296a08483b3702dd664106189e17837c4d630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0cdf3ce821a9f852975ea24b3230cfe5b6598b7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0cf6621fe96b37d62011dec11b669c667b3e5592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0e56b1581438be404e5c244a7c41907ba3db6672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0e8521b7831d9cf02b201fd32a6f2215637bed4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/0f989922baf8174eb0b4fd9998d96bc9cd63defa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/113a38ab8d383ba36b69d479cbdc14a3c32710a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1180442f548bce9b7ea51cb99adc2f753359af73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/11d47d91ec2d93f1b762cae46abb74aaec05ae3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/127dc95d69ab9dbd96187861e97dbd0823c62bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/12ac7f006048b77926c9c57d7028fb625b4a2361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802764752.1/warc/72a36c1c674358c761fdd4469f8a8e1146d8fb0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802764809.9/warc/4a03731cb04727379f986006827b095c2f47bf5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765002.8/warc/847bba5becc34008c78e6eb1dfcf44615adc8eed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765093.40/warc/11706e24e74f8cc6aa5502b30dfba34d6a9768f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765610.7/warc/5a6a5cfe8b8d4dda5137414d406932ff13fedfb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765616.69/warc/041659e6587ed8331e87d37d751adf176cc32402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765678.46/warc/35639c282755c13e6b2d3be580fb5381ebd20abf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765722.114/warc/2acf35dc5c886be26df4ec4fe5fb56cd02df0c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802765846.54/warc/ed4972a34ef564ed29a351b09e82fd87582a82d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802766267.61/warc/4fc0957b4aa36ca23e0b02cc512597cb1149572a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802766295.3/warc/a1628fd3101013f78b69ff67cd809a908571c802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767198.25/warc/634e3006a2721dbad80a5878c5850c923eb568cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767247.82/warc/dc51b40c40ba1248710fe82ba54e9ec454603515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767301.77/warc/8046b1b6050e796b667116b72c9215b3f56eefde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767453.104/warc/6997571835ededf98b8d41db101c978bae0d08b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767828.26/warc/f827d1fbc00e4d06e32bcc054c80136d3c750353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767873.65/warc/da442139046756d0b98f9ead8d944353daa60e5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802767878.79/warc/d7ae2d68016d5662847136e6159ee6809eaf43aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768034.59/warc/301c477411e59f7a268f5a2a910ac65215cc28b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768044.102/warc/690b1fa9074d93a7908725b1efc2d635dfc45629.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768050.31/warc/8e44c2d2faf29ea6403c0d6da3a5dd215cf931ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768089.153/warc/12f756355709c2a4d7aa60099ddd56eb92f2354a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768167.64/warc/a745d9b6d9ddfbea62d50f5a4628b897ead98565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768169.20/warc/574f974a3596a023a13248d013edbd778b8ee573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768205.146/warc/9bce8fc3ec441ac0b63d62be80359eded24bb266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768208.73/warc/448c0db5830bcbf24195dcc44da8dbc4d530a8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768276.101/warc/c8c279ca2d5700dfb49726a2c5459e2217204719.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768309.18/warc/42ab2d815fc78bd823a1f1f25cb8d0285c6934ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768352.71/warc/ba4ae260f6be95d44c5d2795a4c58a0d19314358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768378.98/warc/44e5bcbf9377864e4bbcedbef9e01499f5593595.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768404.109/warc/12df6144d92c2f3198018ea523886cf8a988b8cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768425.151/warc/497eee245b4078a29b8285eba1ac2b98742b8b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768441.42/warc/1882cbdd50768a52d2a1ce6e94d04db0b2ff443c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768529.27/warc/acf7eb25d15f5e2d73085d27ec49b63de06db13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768561.127/warc/402439684009fce17728d3b34857253ef9f16cd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768636.90/warc/e49be0e7ae9ad6a579ef3ec33f3a8c0fbe2cf17c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768741.35/warc/1ed843bd2471da0561c7361b3b9913a0d81d9009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768741.99/warc/d966e21418ba3c4e3551edb521e6f12179007f44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768831.100/warc/252079b4249d8c6d69d9b805bb7b36b089978b3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768841.115/warc/01900fbffa9e67f748bda31751994289c1f5e4ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768847.78/warc/36a13b14afd6790d65f40f68f5299cf972aa245f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768957.83/warc/60ff6672efcbb9f1b0fd567d51fe7c827a407b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768977.107/warc/fca787a49da066d2a0eaf16c7f61faa15484e6ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802768980.24/warc/0f2dec2e7ed06e74c63fa6cbc83ee71a752606f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769121.74/warc/00222988fc437d7071edebbe09f23a524185e94b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769305.158/warc/b7fe2a9a477cc78ba28d96ae42bfac8f702e6b7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769305.33/warc/7026b22d100adfb4fbb285b3a3d8b4644b0ae272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769328.92/warc/d3857374aad78117e2659588ffe31d885dedb7b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769373.55/warc/ae33d65828dbcf3557b886ccae698a8a4d932ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769374.67/warc/807e5cbb5b97ecc8d901627fabab743cea659498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769419.87/warc/6892a619c2d40b31a73173b6c753db7ca9d64714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769550.123/warc/0a2a7015a9204c9dcd7a11697944233d6a9e7388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769581.93/warc/a5ae7662483f6451c30702b1b8ed953d201d8a30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769637.44/warc/6a9c318a2d06785ef56a1ebd5d3b30c63e0fa710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769642.136/warc/db7add047ffb9e66b8579ebee920d1e81faf5f4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769685.0/warc/2d2cf7c35dd6ab13da44f5fd7db3f9c61ad8f237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769844.62/warc/b81addbab51bde55f9bf0df12b5db7251c27f3e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769867.110/warc/4723e3c7fb6a8a0239982b9357f51591ca4dbfa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769888.14/warc/b40bb706f9311f9ac36d23693e97905596afa84d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769894.131/warc/fc905344076802dab3ec393b10f450ebf75800ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769981.58/warc/263407fd6da11de19e6f6aaba99c8168814924d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802769990.68/warc/baacf0c90a8146538b22a7bc28def78823d38695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770043.48/warc/7987dfad7d73be1ee828b198d1dd3921f8292656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770060.91/warc/6c1747f162e0b219527b4bf9b54bdf72e2a85f01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770130.120/warc/f539ec45e108ac395c571e018bf5e571263b93e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770324.129/warc/f4679527f65267c4054396662619f67a62bea0a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770371.28/warc/b1e6e75a6e0c9adb02deff801a9fc07bc3cd98cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770400.105/warc/89f0e5ebbea01deee76ec6e8295d6eb3c79960e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770403.126/warc/e63ce4e433a197cc47117d776d9b5511223b920d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770415.113/warc/9a1a6788c2f009204246090bff06a0d220fd455c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770433.122/warc/63dcddc61566d70e8a47a4dd9d9c5264cb05d60a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770554.119/warc/1857634dd23392c9c4fa1d1e1e2ed89699f71bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770557.39/warc/d4eaf52a3473e0acbdae848c3e4499ed5245bdcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770633.72/warc/e03d1f50d0c7c0e23c4b5a219a5fb4bc421e5d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770668.10/warc/8e7aaebec21adef0176f22136c250765ba6dbd8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770686.106/warc/787d84b3c81d9aa3513277ccd80ac5dcfad0cf3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770742.121/warc/ce2bd21ab142e76c55d79e765f34dfbff590b32d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770747.108/warc/99c0ece9b2266bf1a318f39c303ff5121a168b0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770815.118/warc/bd630c4ba82590d4c0493564e237b26fd84ca635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770815.80/warc/c9ab1c10696ba9d08a8ca33911efdf80f140834a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770829.36/warc/5ab070506b39eb5181a81c84da8cdb723b0ec807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802770860.97/warc/995e668b9d6b12759e9bf242b658e73ec9c6f94f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771091.111/warc/58d7a4586b8e57e73be786fb45a10bbf963cb143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771133.144/warc/c3493227e3788ece64722800b78c8312383710ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771253.22/warc/77f0d98d99d96a23b4b97c8cff055700f23bc2be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771374.156/warc/fda0a24df4e440be334b4d3fb231ea56113aa7df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771384.149/warc/931161de970f286852a3bc2cee8a644584e13cd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771716.117/warc/7436e279f5f6730b1bd2a64f5b2255b76ab8bc87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802771909.45/warc/2511a57bdf137c36fef12926a4daf29d8f6742e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772125.148/warc/86d7a91a89179d760782807f00ee18b44d6c79ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772265.125/warc/7562ab0d8f0e4832a01dea897d56c79a5f88f20b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772281.50/warc/2f6d1c9f20c41c2d02eab6dd305c9042a23b8160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772398.133/warc/c8576be77f95c9f4fd55c30832aa06535edcebaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772743.56/warc/30630cc66f4de6728026514a04b0fe9344f3ca7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772751.143/warc/01ccc2509d3ac2520831a62573f86325644ecb66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772757.23/warc/007d83f4a80a8498aee345ba575fb86b761a44f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802772972.2/warc/ec05f8c69aeac43019e71811c696278b36ee35da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802773058.130/warc/96fcbec2f838020e91b9e6e18ed9f03937924b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802773061.155/warc/9baf1440927bc815802b3318cd709d77791fbced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802773066.29/warc/f50c53bb0cf64f6e4ba920fce1b683e6240ff5d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802773201.145/warc/f2b727fa447f56fd75a5959796ff3284a7d7b005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802773864.47/warc/06e14b7e1726d1b3bd42f92348f8789b16d42899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802774464.128/warc/7d4ec38a77f51e081fc8ef88772199645883b8a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802774718.30/warc/3cda2b774ad4ed6de547a08d95cf3ac35a72f759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802774899.57/warc/1866c2a42d03158b9f304553edb5c09cd5350fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802774986.137/warc/bc6b760af9b8e3592a3150f364af7dab6974a6b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775080.16/warc/78302210fcd93890c2a9718d4d16885453993186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775085.124/warc/98b8c8b2fdcc15fd0f87e913be512a23d751978f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775221.51/warc/d35705a703cb1b6e44e66a8102bdc7a095733b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775222.147/warc/db8aaede143a86b865c8f543c78a4ba24cc57457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775338.41/warc/dd4ecaf986218908f1b9e92904b73a6fa56fee17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775348.152/warc/a31c82582210ea07a1638a80a756b1984dc7f920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775392.34/warc/d5ad2b8bd38d76c2913091f0c1af0ae92578d3b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775404.88/warc/7dcf3e21ea35d2fc6352bf10e2576a6d8fde5e43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775517.52/warc/70e3c35614396fc9615938459e3f2495e18d5b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802775656.66/warc/a6d4dc38f85294c2a540c3f34d318fac5bcd8e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802776528.19/warc/61b869ade791169c39b805bbee35f487dd658e60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802776556.43/warc/d9032b7bbf5b68944bd822ec6a70f065303796fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802776563.13/warc/759ffdae9675f6f0348479587d05e81e599bec4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802776996.17/warc/dd6b4be1d3a132a4c3b98cfb7970506c8ac3e5c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777002.150/warc/e140d9c57c11613fd0ceac8e427bfee674b215c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777118.12/warc/866ba61d26bdc3a4942b43eab35e5f35552db865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777295.134/warc/f1baf2bb3811567f161e32cb7bf0b386507d5807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777418.140/warc/b3b0c68066b465864e6f1955a70f726dd3e73567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777454.142/warc/c46bb7f201b55956dc53e20fbda943f0bc0f834e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802777889.63/warc/096284566538927f360977a4a44870989fa450cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802778013.38/warc/7048c4bfa751dae6e5deb95e346be7cb29480fad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418802778085.5/warc/d5f11051a9523a95ef5fe31ed97fc4472c258acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1418815948154.77/warc/1229daf205c2d5bd38de3b3c66fe0997bb08a569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1419447563504.69/warc/0a3fb8ab840bbf2698e433fa3fd6001ceb3daa72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/146f6b073afc6328aab3c7fede1c9a571c7b6936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/14dc61110fea609ca094c60047745c512a449173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/151e37321b571dbd585c95ae542f0a30242f6d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1652efb629047b0236d0110c654038b293362fe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/17e3658dfb421029a2f8ed9f72560d454691d09a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/182c74b873443b2ffc8fb0eb2fc51a5a86a20805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/18afa6d791b49e1e4af50a4b76c1e4771375193e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/192a9ccc7feefed1a345f3d667fb1b257e8181b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/19d28ed9bf5f92c8d4c48503cda359ba69c43eab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1ad8d5b7741e5f8bdb51de28ea4cfc2baeb2f2d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1b0ac0496b01cde648275acfd669c0a3c5cf2526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1b18e16cbec7f8ff94fbd54e01a55de476e32664.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1b37833e7470a902557c7e6172d95f5ea4eebd0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1d85a082495718f6e03cf6c2ce98ba658c457a1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1db6158b0f0edfff34d806b50c5959a44b79e296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1db68a27498b89597fe308baf2eb7136bedeb652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1dcfd858614de4e7616e310a18745b5ac0bf0096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1dd7754e64bd7b9e43c83391f4a0b08f5238d877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/1ff86700fefe994cbee695aec0f93dfb44a12a0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/24e44d15e878dffae3725e092ad276e0645b7908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/24f408a5eec3e32da9fb28f12d203b916d204766.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/25264948042c93b70e60b61cd8d95e6111573bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/25544d0bf86eea0634d39ac6dc45b9ea8cb38770.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/258442d37b319dcdd90441d98fa5cd8bde242bfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/26b6522ddefcd6f3d48bfdda8daec22a53e21071.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/27313af9d61b173785a39673261318448fed7e7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/27d66e2ac1982a6d3d60b2ad3821b6c7dc5cd038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/28edab955054fe262471076b9b153aba22d1073e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2a209095ad7496c663e83134acf10959c6f9352f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2ab4469ec442e1d70ac386c176dd7893bc0d92fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2bbc74e4a0e044ca6742b8b3249265d689c6bef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2df17c38c399a300392ad337500e89567e8e716d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2df3a9a950cdd79dae38d65f736cb98f0662b3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2eb134a0d76e2e3b48632956c14fa89828ed5bf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2ee62088cf967b91d050c810d51fdfe9fd9884db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2f7b2e700590d0bb7bd141f38c67d363e0070f9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/2f937f235de7872b0114d99bf7f1942db93fe842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3130843503caeee441b1fcdaadf74da247f10d0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/31f0df61f9e21b4e703907b6c4b29b2931e6d2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/33aeafbd301eb1cb332f41f3e369e9504a0ee79a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/35ac152525b3a32028ce4fd8b1e1166a04bf065b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/36c1c737c8b721bdebb5da04eb2e9262607d19fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/38dd6872cfe8ad87bb2cfdea7c4a06fc03f05739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/38e1409d3896f546c67f2f0a1fc4a05a91bc4e7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3984e75acb00306109fcc603e593cac1aba6bf73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/39daf3731c647279d097fae3e8241a0e186bb05a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3a0204abb39b60d248ea436541bf899b2ee3b3d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3a8f5c3a2bef95453fbce85bf8a04d903ab9566f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3bac0a8b6560f8e0d27d90d069dbf8b5eb830b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3c4dbba38cf7596d7dd7a0e20997cff1b9e9922f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3d49958d39a8458f0acb43d24ff45429573ae96b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3d54524b29495ce54af66f611f5124f58475eab2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3d709e9f2e7f81b230419a272ce77588fead61fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3f6ce0203f181af3148be364e24fe5599a2c6efe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/3fd72fe3cb63ea8b9dce2198fd9c6b57c9a9397c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/40ba5a0626474929980e9e5cdcb4cf676fc70bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/413adb5281454cbfef58269518117745dd42423c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4205cb5ebda0206407bc454e6d11e889887d8f20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/423109e760a92371eaf4daa4a93a5eba9b936f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/43966f2b0b2f2f6eaeb5fdfd5f068bf790c00731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/43e858e0e4d82e61acad75f2d487893cb4fe9557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/43f47e390e402d95ae5635f861cbf5b3dc447dc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/442d680d76cd3ab6cc00447e7aa6755c50bcec1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/44489e2f544bd9ccad9229c96d2b28d61ced4bdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4733322dc610e82ac50a826b08d33df62015bc72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/474ba3f886b47864d0d893fb3c297261fbeec7ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4755e23cf11d942eab9e957660cb9a533e5b4ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4882177932296d11579e267ab0d10424f1321ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/48d08c7209f26e445cc040917a4a0a351aa84054.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4b3cd52691377c692b37c06f175643588797ea63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4cee4bcc258b3e4e04e4272b0e27eaa1ea3bbbc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4d574b1326653c4ffdf98502ad66fd09e05e51b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4d6fe4929b4d175510e21650e2dfbc567b9d396a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4db0674119146950d24a6869c8203ffdc09c96fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4f5df2fa857a5e7ab8d33781fefd1ae8a8819a53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4fb44aff1077f99d651b87f126e24851e42b385b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/4fd8a456867708b8827451005e1a68c7655e14f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/526c5186ea14848b38d2b21b944e30547fcb5233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/541f848a24d40481eade2ebc605c01b4a66e892c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/546ae8882dd06076fd80fe1e8cdbe6fc9857570a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/554be2de596d663df4a5230ff3a153216b9687a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/55b115be64c92166e424bb06db8c70cce9894460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/55c6bbeff3e54574ee193e3eb72b945b3bede60f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/56bb6b9b44ac562cd7025a959c02c925ec679cb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/574457af0b0b618cd39abbf43637ebee3a20371a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5867f281b48b95102be6b6297e1babc2883e6b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/590a8052fd64b4517e659a00b1849ac51dd93d31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5a71d3a42f87bd7bd6fa68fc949abcbc51e16376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5b5790e9a144d813e9b0922c4bb152f1c1660a6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5d3f0829e8994e395e578d10865da7ee68298feb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5daab5efca89701aea0e86a7cf43b32f7261ecc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5dddabd7564c627f8d8892c74b39ac08b8df2711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/5f9ed9bae6e8b59542cf925e0f6c354d1e0dd4fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6031ce711c61091081e46d14c4de50271d554c07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/60eaf1a93e0ffd30d0ba1689f170d36a3a2c5be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/61611ef813d69f862cef0eb6907a1014ab97bc35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/618f6b2bfdb8248a27306312aa4bb327cfac5bd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/638a4be7fbc6aa19f568a2bc7e89d893bf969c4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/63cad76d079a0db13d3db2bac810dcd988eaf463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/65b4db6d798eb44ad81baebbf86b5ff2183febe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/66adaae7b3256079815fa1c033fc86323067bda6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/67bd9fd9912bec0b03991d5f4c747dbfaf10d7e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/680774b5a8ec4b58f4e9f7831b390d6bb4c7baa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/68650428b587cbde3668fa31bd3aa2de4d831520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/692eaacdbcb847be20a88f3d4b25b716639ca03c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6a9d560a6a1f7cea103cd008e8ab9b22767df2aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6ab02d50ceda12beedecde4292efe8caaf7d3aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6bcedae138d5803422497c5cb4bf5e328676a408.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6be94aedb983a785651d3cc33d32fe44d8c4f6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6c1ba2ca49e2477ce1a053bbd3868724f26d0ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/6e4fb830cb19e900141d89783ea0557a33f03b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/70b592150a9d10b9982fbf2f94b49f2c34d0d3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/70c19e72e7bb97f345851d4bf4ce16d67dd61b38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7117471cefbdf82d4e722b754d7dc3d0e5134368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/72119e365c6e39b7cf7ec8546c6a5c36da1655b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7480e7951511a7e38e5655cc8c05896b38a26535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/74f3c59b8d3627580daa826db103436df842c57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/767b03db2198dc6a5e85cc1b518985c4a510a05d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7aefb6852c9b2bde4c7c4355b72cca469302e6d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7b4ca96bfd0c37c627fd589c1bd59a78f7555f29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7cf6d16b60b5c3987431e448dec8d99e1f533c1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7d31f7c1bd8d2ebad276932bb4372c4399be78f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/7fa43a4b7eee6335624bafd70aea9f96e5876831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/804171732e382c2ef3b1d96dafcb92efb1458611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/80f14c74c7cb22c37d352b1e16777a89d94f7bf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/82461c3eda042c5070f59ab5f6703e4b693db7b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/84039ad79ed29c8563fa868817c82fa2b8608b46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/84944aa5ea379d5b5f350d2a88ccad9575bd9ff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8602274cb1f008f878ebcf39bb905f454485d6e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8720b115f21ade879e8cbb1cac021de1b09e2a4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/890b98fb0689d139c7bb7505a1e4fc9cf37b61ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/896ef6d94bbe1aa62fd05ac517b1cfec47d058ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/89abfca3457c08f716f2d24b8ef5c30e9606958e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8abe63f3a1fe978a0a7118e6220d1bd0f4295308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8b7bec5d058a42909b45068f7ec4680a9cf14655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8bcca94d8804a899ba06d0f0f51c6092aa90adb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8cd3befb03b8805ea40e9013aac6f3671ca972b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8d164a8c8a07fdcdb5cec1a6987232c353dab729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8d9faabcef2c08fa63e41babe2a70deeade1370a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/8fd4e615fb39cfcb9eadfe3f9374f954fd191b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/912e509902130f358243c122df1f41b336fa33b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/915bea5d7770efa7e910237e00eb64eca38ed940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/91ade420b0277ee38a2b6a7cab6163d2a1b21947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/91e66160c9c84c0fff59501afe6a229898b90940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/91f7e1ec52882339f635033aaa09fb3524c3231d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/93abfc132df720594f9333fd7af7eb9e68981503.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/93f100a34a402338f5b8da88f881d41013c4fef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/93fc24f7ace8f5f65ed3565445d17887e8c1e886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9413f4bc1fa3aeb99ee396212816870218c06fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/949e1ea8d193bc05af54a9bee84e8e02ca8d8f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/94dd95b6e0abb5e0755f17d5545162a7bf8945bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9705b8bdd11b966f84ecc7949a58a4a3afeeebe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/972e968b6dd2b8b4b37401266bd3d4bc53f444ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/978d51b5ebd979ce2e482b0bd5fbad724dc98588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/986a17e3ba93488550e8d69a52f6db0178e7fc8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/98a77c111669339ac0586349d7d56c5e464da25d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/98d58801981a0c614e163500ebb925829d95a54b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/99bc5cf27cfa3f78e7c1e60c7ac993db5b991017.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/99d398459df78557b50704cae9360a4e37c95392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9a07fef0ec520b0d208b39a84655b3308110c2ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9a5bb80f371742677c4a52e0f531217861a2242e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9ac4bf79e0e5152596b049f1515b5e37460a9057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9b66a2aad20b8da335442345e62882e13aca9504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9b9abd7ee797f88e558d7077d594c8ad38c3e9cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9c0a3017cf991fd6f36d4cac80cddda18f1f3a5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9c169a88557bee489f55a10c318ed473e8b2695d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9d7284675f7cdc29da0f0169d686b48675923888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9db1085707bb854afd5840fdd143ab3a14546af2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/9e31d8538334f9d268d258e2f6d6fffd561d64e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a14f08ce904a26c49606aa41d6863cff13cf0f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a2a5a74248aaa4f86e2064cbbf940971da46f98d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a2ddb9a1230b14e10237b1c8d27ff7f50070d469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a2f09c29054322bbb92704be9e65e11a41e8817b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a3fc5bb7cc358dc0f94ce5cb0744b75104e75e68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a4ce00f984370b5fc0c0846bb7c33ace970a03c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a53639752a9686093c924bbf22d7ac6ab45c4d02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a5bf0648be1d6e1c0c2880c1f61fc13ff5b79127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a6d8ab9cca58bc4fec3d84211d40a7b0464852c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a7c6e0b3bdd366eca9aa62d2e63cca65ba7c1b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a7d770e061fa7ad848a62055cab8f5c554c8683c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a882affc397ba5f7cd00d95bb1a5b505c6926aec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/a89e26fe7d356c7aeba67939248e6753b75975fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/aaaf35262bf88ebcb59eb4c353753724d024274f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ac7994636ab4edfe7e3f474f1ca09dae63b35a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ac87e340a86c74495e1ee70b884543e431d9405b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ad2b44648cc1c6222c37a4453759444c1235c7d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ad59ad42122d3c49027d7418f4d9c2c41619319b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ad87deccfcf0e42994378f865ea2b3f6c77376d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/af5bfeb37e872f923d52ec8ad649c713d2836a99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b01d154d04c2943589618ae771dead7756760f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b055bb6cee95d0f22be5c9737f3043c65ce98c0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b0c928e5a6e702bb4104d1bec4e7bb0f66508446.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b10560623d85f7ac14999613e738d2898598b1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b1ab8366b3e499ff84d7db96ca186d55331e600b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b1f196f59b5e9cde004a85fdb65c6697199c0079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b22d4415008f6318db620bbb7626e97c11093fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b3a41b4103b6a5eba13ff7c7f9a42a543ae47b51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b3d4eebb75d4b7398bf6be5a0cb35d3c4bde92f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b6a398c3b1bc6693c07e1e2dcb82d1e3912b5ecf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b6aec60a5c63f075cbfa236bc773d3c0a206df18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/b9b7699c6ece6220350a184fcafdd8cfddb82e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ba022b2ae47832c437ab94d7dae6802381cfa4fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ba2587c046e3f279d0a2d8dd6a6a945b3af5859e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ba657cc3c1a83c048bbc7b017865b08b6d5cb7e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ba81cd0e70592b80813b8af864e071c763481c57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/bc077486e55f06a252837fecc7545b3c132d5e7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/bc4d7b986978545411f74c27613bb0d1edd8896b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/bc4ea3b80940c26e42f801c6cb7e585e05219e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/bd0f879ba31163e5199ba8eee9df156732bc93ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/bf3edf001593d1eab0eaca42e829c285cd166daf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c1b690a242c6e9e6b14e0b4d3e5914a76d744fb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c2333db2e8c6b7c70215f82d18ac9a4be2ce95e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c27a9e5d383c0d4be42f1410d59ae56b5344561b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c47aa7e7839e98741b0538fa23b0eff9ee67478d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c47fcdb603e9f85814f8eaa677cc12b545b3d57a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c516680a835870392c4465581bf53522a916d4bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c68f79f842b3b61ee7d42d98c4011d59e187166a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c70d26070b3b58d2dbd68df68799d93aed3d2547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c7e8a5518944aaa2de26876f3823971c0098c483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c809592f0084be5d895de6af5ce993c01939c23a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c8170842aeff59ff375b33c62e7f408e84ca1f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c8e8cb02446f49ce5755898f07bdf371809c959f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/c91ce9b0c204a1941acd716064a9b160d99f4151.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/caeeafc4ed88547cd6ff69e68e4823207da04826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/cc74b1e28c0c1fddaac07b4e5b9258b96e174ed8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ccde4bfbc8a994f91ddbec3ef5c791588a102b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/cf856676f35cb44ac721707cdde54d42f63b3dd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d098a3235afa386c16e072d5461e1382fc1c3f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d1725671e52c9fef8a663b9d2d1df79e3e0eacc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d2f284db94581c3df3108f91c0ba3afb0a27efbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d3396c9f2e9ad51a4a450c38c513899c14b777ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d479330b984f8dd3eb59ac0707c0e52819e14433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d4a8f36227c6e457f75fd4f8401d1ff919f6ae79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d53a89087647e59af02d3648c26b09f36c7cf2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d55555e801f010a9fc2b881dfca91d6312f4405b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d5b9059023d73696e7ce07a55d007f6a00892b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d63261be758655f821dedab25014ce456d615979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d6b5d3f11cba223c365e51648fc2e4321261f560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d7a479a2d2e86b759ce311774a6332534df71903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d7bc67863fe1e8a6171ea75790e00ac61a695826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d7d271eba667f5278c5c4fd9ddbb5872cc341e8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/d925074beef1cb0b5bc9ed481c569c15e4e29e38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/db61e4d0624e5927611d4790702c54d35e1be6df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/dc19779b6830281e6f5de2fa6b47b995a10577c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/dd9aa4b440ce27b4ea5b3101163d3028e8322d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/dee314ee02f228fa2dcd67db39960ede6e5f8839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/deffe9cdfe98fd7012affb452a775b11e5722c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/df94ec3a8cad1d576d52c1fa048976ecc756c8b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e074c5de5120b053f558b3bfe417a0ce8841a52e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e09fbae32634d42b88185525ef0ba9bc2d0879a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e1585070501a83a1e0b3826711c28a0aebe9aa5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e4a7182bea244d7f90ac5ccdfdccec7df9b2601a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e4aabd0ee66bfcdd55443414b83bcc49f6922095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e4bd11a4814ac142e79b889a1f33fc0a7ee950a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e50707a63e25c2137a91226cd4c9c4ccb503dd4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e66380cc519b4b7568972e584e19c83d5da77a1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e7d8b26c980c6a0e2fae8f287deff09b0e22619e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/e80c31fdd1b9cec6f5dc755135243687886c919e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ea6a89ddbb1d9ab1e212b713937f949d04b5a0db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/eb28834c8afef82b307f20c6c14df0eefc95ebea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/eb99037e4feacaf029927a2b6ee31f4bb8536ef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ebd62c50660a6085ba6554046a186b0b4b03972b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ed695190479617f91f834c105684da4ececdc957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ed921e54b3c4267b0ff546ff091f99fd2b62c94a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ede4543fbeb598738b35a7b8d745e87702cc7f8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ee0ff601ef209af87dc2ecc4d23e32135c4555e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ee68a5b7160bb691a14340a5ea73d83d7473e525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ef97228923a17140b7c15239796cc641c47f0e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f0587e66a627cb77a4650469c23031b07c9003cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f13722b92905355f3ed3a13ba7404c51690bd26e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f23c4de9d52b9ae5eb56c1a7df53e405533362a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f3cba8973196d5b1233f070c2e8ca7b15ca7a3f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f4cc83ff5c0c7d83c4c94111e204ee03bf4bf236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f50affa53180950b0eb8cea11cc1f063ecf99ec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f863ffbf67892fedc76257494b3ac381b664e46a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/f917f6a0fcee5fea3177672b19fa85afa125798a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/fa467037c1c6f9e6da62a0d58987b2c8e10604d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/fab43e3be7aed7b1856360994676519afe62950d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/ff0f0ba57b81fadad915996b8ab966cdef4bf662.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2014-52 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-52/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2014-52/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2015-06.yaml b/configs/cccc/mixer/CC-MAIN-2015-06.yaml new file mode 100644 index 00000000..f91745ed --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2015-06.yaml @@ -0,0 +1,361 @@ + +streams: + - name: cccc-CC-MAIN-2015-06 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/0754fa7ebacfbaf6492b30946ce01694a5139a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/07de10a03a03d6b7a34d664212d5df8f0d341c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/08fd245b75b07036d9fcdbc2e07bec91c8f50cfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/0a90927e8b9ab92d842cc884c01c5e8a76095a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855094.38/warc/709e9d200be5d6e5a9f0cc3a8f2e6b925d04d262.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855094.38/warc/e31cb8d7d6635dfe84cdd060f5d075d5da6d48c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855561.4/warc/7603df4d240467f0babf8b624d058479654f5d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855561.4/warc/cb4b8712f1592120aedcea7d0bbbb1b6d66feea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855845.27/warc/848f48d316ef5ccac80b299151e1814088bca967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855897.0/warc/4a01dc22f7380abe2e4978556678932239df4aa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115855897.0/warc/8ec565486e66c4368e9e12e58eff84daa9ba4990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115856041.43/warc/49389ec6ca025c4848286e053972706158480384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115856087.17/warc/a0837f31ec0308f560afe74b7daff73595639c97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115856087.17/warc/f381dc78a11cd4a064a1020c8330ab5156bfd358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115856115.9/warc/a106b685f0b9dc01f7cdc6b831f6af1759c8f59c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115856115.9/warc/f47e4a190030903d4fa9554e75cc60a03a6056d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115857131.28/warc/8e9ce5c9dcaf3f84947ba99914c7265c5dcce9a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115857131.28/warc/a549e9b19a0bf19118212c6f02d11e509d3218ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115857200.13/warc/1cbc65f839b30ccdfd4a3e9b3570d180b0bcd7c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115857200.13/warc/876658c6b882c2b9755a87bafd5bee5ace7a4d76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858171.97/warc/1b65fe9ca66415aa1b9fa6f1bd323f8ccd8b726b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858171.97/warc/6edd4a782f921b39f6138b041ea2cfed59b490d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858171.97/warc/a69ef04a3e2045a8bb6c917d9b1ff92fdc3af3b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858580.32/warc/df912ad667e3e81d9a2d806fc5c4db0e5ef1d58b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858583.14/warc/06e130dc80fe4366b83ce3e3fb2e4e9c2ca3469a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858583.14/warc/de18fb1db73bedd24e983276e53ec1792f30e71d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858727.26/warc/0d58d79352b9aa854f4dc81accd55ecd1a258e69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115858727.26/warc/2c8499b4bae2d637b19d9a0a91ef17627b84d154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115859923.61/warc/25b909110a33dcc93d06f111892ad01a68ace1c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115860277.59/warc/3903c3158e419e9302b16ea13eedcb941c1f243d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115860277.59/warc/87b590b1a04fa041c1dfa667ba63cf9ff96d42df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115860608.29/warc/b376fb67bc48d1ba3314c86e173070525396935d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861027.55/warc/0bfdd9ba670db4513291e824c02ec6a215a0509a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861027.55/warc/53a313199ba24694a0f1ca6933ccdaf3cb83fcbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861162.19/warc/b97a526ef608fab9b0350fbfddc3be51050b00c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861305.18/warc/385ff9a21d43d2d288d34bdb2c5105ff24073f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861305.18/warc/c5ab147fc3f913f413ec39ca4f9eddab9265b77a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861872.41/warc/204c5b8f740bf3bf55a29cef46454a4ad598c73a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115861872.41/warc/b9f0bc5874cb7497295de75fef9253401ba50a06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862015.5/warc/ba55a3725a4cc69e8d9fa825d327813aa18506e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862141.23/warc/7b6e243ac8859fc9f7a9a92c1ed38bb68956e97e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862141.23/warc/d51a08295ba915e8176d8ac711220165af8d00f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862207.44/warc/071c8ad2f0065fcf79737122d00053352ff2ba2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862432.8/warc/23c590af347122f08e8723ab4cad2024cb5f4a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862432.8/warc/4f8f2434835b7f9f4f3b2b9436db9f1106ed968f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862441.40/warc/0a17faa92da588cbbfb7d0a1841d7c0efe7d7fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862636.1/warc/0fd5c7304e82a8e9e7c8c1d9378c2c9d9ef09f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115862636.1/warc/d6a360bde7e8a62bb993d1edcc5762193631fc35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115863063.84/warc/e8cdeab94bc372c5604f9297d2997a2235fd1d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115863825.66/warc/bc45364a018948fcdcce61b337d14717c112d9b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115863825.66/warc/d301eee810e86a86289854e2ef3d0da5e122fec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115864313.15/warc/b52d650d5409f98e3a546d338b89b66780acd928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115865010.11/warc/64bf8406259d0825171459697004215109b306e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115865010.11/warc/71c2ec08d3745b6d408da5008e2727a2ddbeceea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115865430.52/warc/b21c2ac6c29aa2b4e8445b551d9103de7f02ff78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115867507.22/warc/422ba6b48aade8f891df281456bf42aa6b97eb2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115867507.22/warc/990baddaa7ed53d5161ebbe26d6010641afc8e25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115867691.21/warc/c3ae90ebdb80c1dadd8082aac4d90dc8a6ceed2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115868812.73/warc/62d0aa814bd6cdac74eebc15f1c54f8bc971d00e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115868812.73/warc/f50650863568c020449dc39f5ba9db74dabcdbd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869264.47/warc/1f61bcc3a12abb9790e012dede1515238f13402e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869264.47/warc/5b7f5c182cd422b5a0f48ea16d09b185169a9b28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869320.77/warc/f2fbc6093b8e5f222a6b4a480c7ef2aa168f61e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869320.77/warc/f6199e0f71f69d48c7882e4597d8a825ab05f626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869404.51/warc/4cfc57b7f8782abbf9124dcbc5eec722945434e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869404.51/warc/747875fe2630b97bc6d410366328dc2c11fdbac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115869647.75/warc/0b53563cc4be80cc7270f18d6ed75e9293d8f80e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115891802.74/warc/13479c5303fac2eafc0a046c4f316dafea0be627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115891802.74/warc/c18559ea9dbe5aadbf4b5547e99c1a6051abef5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115899686.93/warc/9faef1db4136e7b09460d464e5414a6957c5e34c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115900160.86/warc/04318e8e960454414064b43a362c9c480ddecb9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115900160.86/warc/522c6d76d4ae7d298e25d73cb37f7db67a504c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115900471.63/warc/37d8c40ea20d16f6fad82fd68b40aad3d62784f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115926735.70/warc/0aa7576c2f956f98eac5022315d9eae2ea6b3077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115926735.70/warc/8d23937f5124bff020d8656a473436e28ba9b654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115926769.79/warc/e322e9c1bcd96008d80c881763be1c1023027ae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115927113.72/warc/e8cc98f6e10e2cb4bd8c902eaf6254b3c1d463cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422115927113.72/warc/f1a055208c62b149452bf036619397d9caec654d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118059355.87/warc/888f62f226c7d3e757bbf9159b41717b05ea3940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118108509.48/warc/33d0829a292231ad48a38cedc485f26f2064c6d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118108509.48/warc/5c252f864896dce293407e71ec5381c8195dc9e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118551401.78/warc/295c9de66a911f11bf9bbac20b2435a44796953b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118888119.65/warc/56837eeb506d2e269d0b2c9621f69bfadbdd2fcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118888119.65/warc/8347f76a0d2e8b58ebb3af4dc62e5f4859b5b42e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422118973352.69/warc/f38c8c17c2e2cc57d5783bc0a63d7ac5deda934a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422119446463.10/warc/3ebc743f140c8781dfd1dea6109dbadd8328b861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422119446463.10/warc/e844599e7be32d09f0ce6091db129085e8333e1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120394037.54/warc/48d9f6ab12c6de57417ea3255f5714e9491830dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120394037.54/warc/ae4539799a843841a47841fffb3999a01f542042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120453043.42/warc/c6dc4d17515362589e8682165e87668f73a901b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120842874.46/warc/02de22e84b13eda475c2467de9b796035af324d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120842874.46/warc/6a2a886d052536eb5abfbbfbf8ae3895cbeaef92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422120928902.90/warc/ee6510c447dce3bc361694936528bc905bd199da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121478981.94/warc/3be166222fbe00f670f4fcfcc0177fe4a1644284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121478981.94/warc/cc8d4e19ea799bf62e2c8518c5e586d7b3efbcf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121540415.91/warc/dd2102da78111a67156b59c74b27aae6904ae6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121569156.58/warc/23e1ef2d8bbad26b35c81eb2da39422f7f27aeca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121569156.58/warc/c6327b11725bd69f6ece030b4253be3f99c140c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121744242.57/warc/d84914b121f3ee96f918b7c7f7925b133e46a73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121785385.35/warc/3f65492540cc3cd96adda9e56badc10e0051406e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121785385.35/warc/64ff4342dfbf31003b668e47f00fed8eac677333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121833101.33/warc/c5fbaf5ebf77f7199d543d7f37e23aee86d7df05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121899763.95/warc/169c48418a5999894cc91d360145227fff4fd35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121899763.95/warc/6eed14320d7d38bef7e877692e8a5928eebbd65b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121914832.36/warc/369c41b8aee14ce248007bd3bf56dd46dfe538a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121934081.85/warc/6ab99d2efb25cadb68613d8a062db856c09c260b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121934081.85/warc/e2776e30e027932fa19be263a034cd79f6a6781b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121961657.12/warc/b56123b8508795c8b780954af6391afaff9a863c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121967958.49/warc/544e922fbd3b6297bd5d35ef0d53a5fdd514c319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121967958.49/warc/f2ee451a5958f0ab3e1f0b03fe45d3cdd31c3bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121981339.16/warc/2f86c3ceb0f9c334d548381675b78958494b05a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121983086.76/warc/3498b65164b21d4d27eaba40aa5104b3cd892137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422121983086.76/warc/8d675e64d096df19029e6b5191d86f7de17f1934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122022571.56/warc/2a9aa62a2fec9eab9be35b439337a6dbfc03a683.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122022571.56/warc/32ced2bffb7cb17a8e250ff35812f0736d131a53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122030742.53/warc/cbdf60f0daca7e69d1465c40643b487c5503ef2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122034298.3/warc/3b98d5873b7ba2177796da9cee6210dfbc6a3652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122034298.3/warc/a09ff606cdcda85e4d7fac007b6f43fea0a97a58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122039674.71/warc/14d09ae22b497594c993c50a61e94ce7b09dd6f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122047499.45/warc/1b7a112643793da50d7fd10ee121586708720619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122047499.45/warc/32a2671da6ff4dc154b51bb8ebe3a9ad11b4de67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122059136.7/warc/43f307e1c0eaa5e96e02db9778c249c45feea32b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122071449.31/warc/0d85193816402c88bf3147a8d26cdfa23296a735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122071449.31/warc/58d1ba07a42a8c1c5d0f2c5a168e329c6f700820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122080417.25/warc/d2042c961d0e952bab6d4eea89718d0652a0ab6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122086301.64/warc/04e0ece447ccec73edf1a71614b50ab5f7114688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122086301.64/warc/4ddaa9f5e1d886c5954306df3200ab567b76a1e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122086930.99/warc/186937236acd68b4fe904225e8f72abbabc4f5ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122087108.30/warc/4204933098d13841c11061d70bb8dd8b0154fdf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122087108.30/warc/9f3f3295563b93ce68a0319aaa6859a2f1c10da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122102237.39/warc/a18d1315bca90057504b9da9d05b368c216cc6d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122108378.68/warc/c4139f0ee8c8139eaaed8a476566baf72dc4da46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122108378.68/warc/d0ef7ef6c8a342df5dd77a9d86efaaa688631141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122122092.80/warc/4bd94b0f5483dbf3382d0ce7cb9762092bc461ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122123549.67/warc/15dec56b646e04d9dcadd68f67a67ef07a84fff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122123549.67/warc/6fa750bad222431599172b55925aa8ff302a82ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122127848.98/warc/627065eaa86e643f42f5add667571d7cb9b5280c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122152935.2/warc/6337a6b04718c34338103c0858c3868e394e18c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122152935.2/warc/aece40e2afe5679f962e4acb2d85236feff6da0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122166061.6/warc/2eaad5589714462b6f83467ccd9db962d396e426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122166061.6/warc/d947ceb07e06c582fdc577f1d59365062e807272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122189854.83/warc/0d6267b4221b6a4f629b72e18813f353263326db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122191023.37/warc/9d6c8a1b61827b93ebe169bf862ecdc46030caa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122191023.37/warc/e102eac4959924498a2d090d5d42625e2a2be093.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122192267.50/warc/60b4abc096c6260071295ed24f6d3708df3316fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122220909.62/warc/2474af3a4947e2e0b3221754c3bcba8287bb3f56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122220909.62/warc/bc8368aa44987f65981381ac5d9db40a73d7fd01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122222204.92/warc/bd9915cace1e0ff4dcafb09d0c62f46720e36c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122233086.24/warc/15b3b688a5d09ecfc16fa1b87707d0e230fc2f85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122233086.24/warc/b4dd27633553edd7ec28a16e516fe5962fd17b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122238667.96/warc/0984ab16b398c26cc98fbeb5ad1840ad2c8e6db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122238694.20/warc/7aa3d6ef49055b57b67bbd22daf6ce8b140c8f91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122238694.20/warc/b54991f440dae8f8855653068b6031b8e49d20db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122245449.81/warc/b2d7addc137d44aded144c2897d0fae5abb129f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122276676.89/warc/164681ee0e20669b537bbb478b92993ae93d4fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122276676.89/warc/a3ce7ad90dda061cc23b12ac58195bc647b7dd36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122328486.60/warc/95e9bc6d4121a807b00298209b392e7e532ab7a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122691893.34/warc/66fb4caa88af41a68d6d8a9b14b77753cab0adf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122691893.34/warc/889dd83a0c942b2fea3df20065c20035454f2b25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1422122691893.34/warc/bfa5358f7019f899fb5133655d4507e468dd215d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/158166c2b79e0aea60a64ca04b45faf67e0b0b98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/16370304e95359c6d6a0b9342e38d1908ae5f9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1a0209fe9120a2d688217a2206ab3648090fb118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1c3337d3cd3e2370df726f14f54d8731a638d21f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/1c68b8bc52fa5e1831d2ef41673d1ffc987ba1e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/21654e3b65977b697d1381491ae6fa7934c84e1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/225ece17382eccbcd2429f65019084f72b5fc098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/22dd1a78b8b78767c28eb4fe6de0d05c19288f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/23dd0a7193c8e191d5ba3c5a400cb6e63870ab9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/277ac22c55938f7b041cd391e5057c4dff5ae02f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/278106f126a2a3b0f0ab9efde1a4ec81b80e1931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/28d28be653019476311c18cdefc7c00579004368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/2dad26e48a9841ee935722717be468d8b74b3fec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/2e7ff77efbbf2077e36d430ed2caf1379508f789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/2f267a0ef3b194def68a401491ae8efd1883d5da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/34000a5c6e6d160c23e9cb27ab3681f838dba00c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/36b471f74c3874ee946bdb122e51ed94df9cdc6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/406cd65dad8180c9b24006f10ab458cd453def47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/43656cdadbee9f76bf064e0454eac2273e00ec7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/49df37ec74bf838ee37a3c18fa1f74bb3f0999bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/4b5c2ddc36eae13f69012343353d4738321d5f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/4dc3f971f41c98bfea04e420925d2212af43ee32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/4e84b6aee538e486a151b3dc2a341f5956222ef5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/4ebf76a97e6fc4f02168119c5f940b3c36b39500.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5310c5d68b6216f0267ea2cd5ff63ad8f9cc7aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/55f3aae2dbc0876a0d78dc37ff96f366df9d96f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/564ed3da6f8cd4888923ff2d97c82e21befbc679.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5731267a1ba32c8c2bb854efaecb2a87ea4172aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/59bd6f20d6687932f718d8ba71cfd711fffa8978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/59cf3bdcd687ed617868edf498a3796762e75213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5b6e7ec0cab334ff0292c5db2ec4e827edae53ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5b99d64bc0173fec02965b193c99f1569c4948bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5ef39017b260cf8c9a584c1704e2bde4af84d61f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/5efc2ded7211fa76e860fc8316dd9a53b4832191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/639fcf90deeb9bb2c3b4e3ca0cbc3b59f429234e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/666f0ae068ef4e2b3c263305ed50b6bdb413f64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/684c580ec9e0dd259ee3044cf1e65b391197d8ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/69472a513dc7bc8ab018ea7e1b008ce4604634f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/6a863513ebc732bb7e50992b0d3e9969bd00944a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/6afad92c0ca1dd297c845e77497abe94274d213a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/6d8cdedc5e6419e5ba65657eb505b1c942dda0d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/6dcb52e0530555eade421c6872e2a05a49f1a2a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/75d44835b7afc5d82bd22b7a273c3f1bc2fca668.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/7bff4920ba46c8a1e706e1a54b0b8708d9845e18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/7caa2cca421350dda82464a73e43b3b43853ef90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/801dd04b0179c2243f596e48a88e4bdf4a1a9da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/87594d5c507e95fe1a728f5fc0e5e99e70c83494.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/885702d2db68a0345969b139103b2d5f622f3e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/8ba78ecc2d5c3a80fc7d0d489a36483718c0e257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/8d757321184032682ff701352b03329e62739c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/8e59b7189b3c6093a11c738728aeb7fb1ca05f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/903680602c036a590d7f9713f02650737d2a56cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/9276e39a1d1167c147788311f5e7d79f3bdddc82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/92f1941b0f001f9afe0ea22995d45c4d623275c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/98667ecfb831faa99393f487ccd2894de701acd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/9b73748511be228de9f12ebdef2a1a838a8e44ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/a082cd68ace75d5ecbc9a9bdc6572c3d652287d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/a2e5e32c0bc113796fff572ea80754aff0e93c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/a4b708354159c7d9ed5c0b9ceb51b7bec691bc65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/acc2cf3ff61a877870b17a844e3d46fbb35da07c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/ae5f85e6fd21b337f4346f7ce70e9b6ff093c81e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/b3ee970980d9f108e9390330429c781fcf82db04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/b86eb119e35a562eb39de2642a3a32bb9fb70295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/ba0feb7f1f717aab91441cb4e0909b3939b5f161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/ba3e29398200f664e82ba4677d676ad08ea24e63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/bcabe1b4626a936282fa773e96033ee6ddeb7951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/bf44dd8b2586f26138521abfe5056df82834d53f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/bf72234a557b717fedab761db39b71e6b4119297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/c4a9967a6c717a4d7e5d8549992e5049a5cff336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/c4efb7f3f3bc3977f763c937200b7f6101f7010c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/c541f330048050fd45ee0b64526ce187a0fd2bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/c683b1856afef520c03410bc27e59c2421de5905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/c9a36998739422f58a3adc8a3582641e26429cf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/cd66de1c179369476592f4b71313d4b80291cebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/cf96ce91694bfd8be393ed1cf67a0e5c5c808ffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/d21f24dda585b065db7e7f61c3a120a2156d0ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/d52525db3d3920f70a368349ab184550039aee43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/d6f6891ff56a482167505de44e9eedd82281fae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/db711696d055ec0fc4e5134476651d4ad75b160f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/dc69088b669f14d604953f3da38e11d233577ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/e0850ed56173f298543ee11713d78967fa1b90af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/e71330d8bac08d1308536e5544de4a3c4cd84b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/e862c9a67cf74bd64580c7ba40f81debc3500568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/e93350a857cd7ceb1c843be74b1f9f39ca3acfa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/e9620adc7ea5a7fd04968b1f563cea6bcee0c380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/eaf0e4b0ac2f1b5428425a15578afbe61bb19921.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/ed9ee2aedc99309179fe63766cbe6c5796fcfc39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/ef4ff2049045dad73395214de7fddd9fec0c48e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/f29a642c9a6f32a3ccc93e632beb43243f534fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/f5961330872024f2040391747577d8db92e6c9b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/f76851eec04cb2ee642469e2bec0796816a2bad8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/fe39b38798350e8a582a632c3cb938ff62b5d522.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-06 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-06/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-06/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2015-11.yaml b/configs/cccc/mixer/CC-MAIN-2015-11.yaml new file mode 100644 index 00000000..d8cadebc --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2015-11.yaml @@ -0,0 +1,440 @@ + +streams: + - name: cccc-CC-MAIN-2015-11 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/07f33c0e981cdf12e50fabf7480a47a8b61ce627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/0e6891a82aa3ae1072d88e6978403f6aa29d9fb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/119be949c2691fbcb0d070e45947980169de9d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1206ed7b98e73dd1f98611c18a49f3ba15710447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936459277.13/warc/654ebcdc3014f7c83e024a50a752d6da01ef1f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936459277.13/warc/86d7275f80cedeb0ee3075e9025ff03e3da0499e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936459277.13/warc/ae94287bec2c59d6ab16683e97b414a6337f4353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936459513.8/warc/04e22020fdb184ccffe3d066d477ad35b5a913d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936459513.8/warc/bc1cde275aee99e9b0d35bf7276149892862a515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460472.17/warc/9e1a744786b38ec6e8afb3c7e346c01127fc710c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460472.17/warc/a4cad0cb407a2ed3ab8e959150d65df1c0237c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460576.24/warc/6292b03f2d1de6be0fda3bb2baf5e1e564bc7457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460576.24/warc/84d1c324c3b0eaa0d720a234e6bbfc419d593f0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460576.24/warc/f1123f634458a1ce04847c32e89f165a2441f5e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460577.67/warc/bde02037a27ccf4950fbb2c7939b8e44358e6d80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460577.67/warc/ca81aeaec6fb8609cf532abf0db482ab1a58c417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460839.35/warc/2e34a5d322f5cbf218c9cdc80a867896e14a3ea7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936460839.35/warc/518acae9593a9cefe3493b318c132d09d145d878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461216.38/warc/1fc03a4bd7aaec1c3b7a6d39cf553e1794895d8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461216.38/warc/2dc1cdedf30fd043e9f1432d01c7150f763f1283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461216.38/warc/8c4cba589631ab02a9c756c84d448b1f570b1abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461266.22/warc/32bfe33819e778bec10feb8a94e696e29e0ad6db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461266.22/warc/781e9f97103948fddfcedd801c80756b40943e94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461332.16/warc/dcc2feef79508ffba38a90f9f77d96d03cbf26c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461332.16/warc/fc8ebbb2cf222c6475c4045567a342d13fe2917f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461359.90/warc/e1b95d1414b596b0b0b7292001212901b5d07570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461359.90/warc/eca27bc10b00777f1e365b2193e906ef5f339917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461416.42/warc/4040d8b420ded897f01755103587c87be310a716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461416.42/warc/6b590531b9c87f2b6766f948b9e6634c0291559e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461416.42/warc/a266110415ff482bc54614a21c380f2d6c1b9178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461494.41/warc/2bcaecffb1057f4de6972fd3f18b890614b0c252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461494.41/warc/ad4228aa4c087ce76aadaf1d55de1fa604fb76c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461612.31/warc/a1f68f20075c34ae56a30ae59a1d9309ddcf18e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461612.31/warc/ce2bad2515c1d3893883cd762ad34a8526fae633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461647.44/warc/13bf33672daaf652986d2ba05766bb808f2bcdaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461647.44/warc/b9e8a3c58386c5c3bbd335ee32b86c4014f4f100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461650.2/warc/0e069a37d7303ed50f0051f8e4753946ecf6c4b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461650.2/warc/cdf4818be9ec2329d8365e4c4ba7a5c6388f90ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461650.2/warc/efe38f730d037f588a355588858f2b2a21ab5d53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461848.26/warc/0f1e12c094128dbb0d0f0812799c3f0064dde74c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461848.26/warc/4a92c96607877590961c72d91529134cc6707fb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461907.65/warc/598060e9d4b2f6f31eca024f87cbfbe25eec8237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461907.65/warc/6dbc1c031e2827bd48a143cdeddf5c6d0763d2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461944.75/warc/01002110a7fb426839bd180ecbc0b8849e227ca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461944.75/warc/ca2d85de20c3297241fa39cede8e5106f0e82d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461988.0/warc/1dd94ee3911a79b0d2374ec32ec087a3921c633e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461988.0/warc/24dc1ebac9ad388853592afd572ef61b8cbef482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461988.0/warc/d5afc8783d70ad3915e1708df80521a53f2cb459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461995.64/warc/2e950cb6cbb20cbe634ce4c059ccd330def74994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936461995.64/warc/5eb19377fdcf56632aeba171a4ac400038dd408e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462009.45/warc/92cb73b4b7d3544001cd496d505e9f7fe748c6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462009.45/warc/b99bd4127208eaa8f0f79e7c2a944fe7855dfe1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462035.86/warc/9d0b92dba6ad173549161791212505fc9ba39fc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462035.86/warc/c43c4a2836fb130d5af7516450d45ba3e167a342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462070.88/warc/1242a799ed693d8b19b6c096e5f4f2cef5a4236b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462070.88/warc/4565bb701587aa103b221810a3147431afd8474c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462070.88/warc/96194bdce1598c0d0a8792d654b73d11afbb3fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462099.15/warc/cdf2d9c521ad01c54dcf5e97fc930fd105da0055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462099.15/warc/e95d0bc44a3b49d04c2fbc6b21a84e86ea70c344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462141.68/warc/985a454f812fa72b9d8bac62eb99a6b097ae2b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462141.68/warc/d8ed6bfd062f12fa0013226f18008e45ab9621a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462202.69/warc/914a3901af580695150eba56a46ac11a23447df5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462202.69/warc/99cf101e2dce064af4330467deafb821c3f9e681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462202.69/warc/e59c1572d16896a86146326b41ac559ddc05cc14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462206.12/warc/38fa256787b0850175e4996658ace16453a2ad08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462206.12/warc/d7e730223edd9f6c2283b42e3272359cd341074b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462232.5/warc/0b9885a41f01b8079a5a0b1596a7d1641cc5668b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462232.5/warc/43bb8cc1c0cfda3404d9db3e5c656a366ccf69d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462313.6/warc/60d760a939aaf1d45c4eb45e2ddbc47888e31b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462313.6/warc/65da22589b92362c40d49d059f0090fff2986102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462316.51/warc/0d458ace64d8b73c9376ed13bf068fd7a485e31b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462316.51/warc/b72b8bc91d62b2ba0625141cf8c2ee8ee538c50f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462316.51/warc/bb71bf38eac41425980a413cc6d58f5fcb883860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462331.30/warc/4e11017500f35c136380e8188c88704eaaed0ea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462331.30/warc/6a465352bb7bc61347bd95c3f1117d7384525091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462426.4/warc/086e63727e880dd695298d4f2110a8add64bc585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462426.4/warc/9aa36cd49a2540a0fc9844b5ae8809fe4a1fd61d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462472.19/warc/09fa9b39d74db01b20dc81cec056f23608c3b392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462472.19/warc/0c75ab582889e3c889a681e14e9e1bb20f492aa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462548.53/warc/79f09e062594382264d05050789b7a6748d9371f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462548.53/warc/b1bf3fd600707905de34e2490f867b6ccc8c5906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462548.53/warc/f8c825006b55e6c08bcb6b1b55905932ca0614b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462555.21/warc/9c14529fd2129eb16c7e20cb05cba0f12c6bf5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462555.21/warc/b0de95811b7f98139afa04bbcb579fb2dc556bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462573.55/warc/8ca9dc0dff29effd5d3446850ce18053a910feed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462573.55/warc/fdb322e7cf0c965c7ba9e3202530ee617856406b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462577.56/warc/216c5413c9784ee2bd3c447fe32849b85b4a5aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462577.56/warc/794f1d4a57be99b7d68a2c8db4dcbfece49db901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462577.56/warc/cadad4cb17c330d6cb042e98f773951677d9672d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462700.28/warc/141110ed0023fcdb0eac011101c60051a8120a8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462700.28/warc/6f71b814581663a747b415f80d829a63261f2414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462710.27/warc/5d764b5e51ba91259284918ceb253401e070bb7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462710.27/warc/9aef5f1069f6b77353cd8166bcab60e37c862f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462720.36/warc/ea790a4d913dd490191f162fadefdf5f2d7d0946.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462720.36/warc/f3aebd84659aa09ccb8621f59b8bb6c93765194c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462720.36/warc/f8dae3afb2b5094198021f334ccfb37de2f66c73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462751.83/warc/31ea7de4cc4a86e2bd2062a9d681fb15f0b89b0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462751.83/warc/8a2755a43a0b8c503dd2814b9788ce7641da1682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462751.85/warc/44b5d1d631c9ba4a8af528200e461b4bd00fb48b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462751.85/warc/91e42ac5b8cd1e27e1ca96c1037fec0d168542ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462762.87/warc/3bb9ba8d2eccb04c1408191741c1f55568f1ea05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462762.87/warc/b9ffe42ef18a03112d676e51f00d02ffc4fa7627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462839.81/warc/a986d5c55f1bd4e7f219067d48ae97ce5a7c9643.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462839.81/warc/b19c6b18e5dcc544ec03c4187a0c722ce1041ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462839.81/warc/cd8196148ffe588c219c73740becd987d10a2248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462898.92/warc/2a082eafc6849f885905b13c23687ad416af4b29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462898.92/warc/aa5e29b3f4b9da6177362e94300d7c277cd282ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462982.10/warc/321229ad2030c3947f4cd3dff99a4d5f70a8b1ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936462982.10/warc/676e2ca9b591ce0083dd1361c5fa5f0b3f85a7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463028.70/warc/4b426dd7949e6c248ade6dfc69b5ac4dca77145b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463028.70/warc/7f49342b6770d526d5603a1e4fac4486d88ad8fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463028.70/warc/f3baa5cefd720430532e1f47135944fa446a9550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463093.76/warc/045617e3e5761c8af3209fd1340edf11e716bcee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463093.76/warc/89ea896cdd8eebe6e263fa4bf28e5d043d9fe531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463103.84/warc/0100cf8189498ce59cc5afab873543483af0bbbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463103.84/warc/bece6959552e3fa0c38ff3d924acae626ca96428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463104.7/warc/1ea7f3c7b2e8c98ad9ec5597ae7fdff1457f9286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463104.7/warc/25bba774cdbee6bbf3fc33411af08f8a0f7f7682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463108.89/warc/413e833daaf96917dd1dea019659f33072e8acd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463108.89/warc/57553bffec955e9a508edc8702c4bf7a02be54ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463108.89/warc/a73e585b164a98158e752cc333eaa50cee19146c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463122.1/warc/030bb3c88c1f1bd971536d11b9b16b9188bb77da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463122.1/warc/68aa0b8fc8fa5c5ac75989a162b4b6d437d7b279.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463165.18/warc/d95a01302ae9eae5fb4e1b958e9eb051d8090aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463165.18/warc/e703db8d27d96d92c65cee817bb94226924dc127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463181.39/warc/235373f77c43b7f4c4eb867d03a1adc8eb682ebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463181.39/warc/5256b118dcac3349390078d8d3dec5182d81b32f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463287.91/warc/211860ee26410e48d3dd8a6cb04e540c6b3a7b95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463287.91/warc/792fbcc0dc4b28515609388baab60da3410f3376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463287.91/warc/b533735445c7d1a9b9b26af199ad33127234c376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463318.72/warc/7a6ce94837b25291039ec783d87ff106025b87fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463318.72/warc/f3f8c80ae80545d79ac8daf58271d90444a1b888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463340.29/warc/818e3e274e0e0b745101af34003f3861d4e03964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463340.29/warc/c93f226980354c9c1760b451e401d2b6d4be6f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463378.98/warc/901e2783175479004b6e8c9802736fa00f996dbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463378.98/warc/e833a173411bb3a7279861cfdad7ae279c55caec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463411.14/warc/029396f998695b832260c0c02bfb110f4a99fe0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463411.14/warc/8fae9684167121d6b380a28efc5f8b1a8e16f3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463411.14/warc/da886be41b5f7452ef80cf63a9e968d80a1413d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463420.52/warc/c29ea907946ce3d89273a5e456f0410129b7775e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463420.52/warc/ebc71637d03392653e9df9e3a3926a9fd3be77a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463425.63/warc/53f7e5595380347a823a70b826d4a4e5df16c410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463425.63/warc/be8524904296499138bb34ffa5a09fa717156e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463444.94/warc/0414ef0561b94a663da3c84cdc8add06a581e4e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463444.94/warc/08ceedc05e6f054e929e9e2295544d98335269c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463444.94/warc/a7fffb118ce53bd9236864825fcaba8ac9d8dc86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463453.54/warc/906bbc1b2ceccc99906dc9cc6acd78a48339b83d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463453.54/warc/e0c04eb5e50b11936fd20057ba320da3d4ea32e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463458.93/warc/7895c6ccf7216b575b9f6a450a49157d8f366d2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463458.93/warc/95d7277623f959a0c57a1d7df779fb7e648c0cc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463460.96/warc/8a090663719bb39c38a4902ef1d97d38783905bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463460.96/warc/b8507fb91ae50ec927bf04ba864498cccf973f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463475.57/warc/14277f017132f0a1135968da0ca441eb9c69b34d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463475.57/warc/914fee06a323eb0a38a620f538fb87992b528d08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463475.57/warc/e07e1e9b2d23527c4b32ed7408849f51688f784e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463485.78/warc/013a0246e51ae9377854b6b12b61e9477b675335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463485.78/warc/d492eff3b0c70bfb9bd1183a92191fa48dac9da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463606.79/warc/64e3d691186539f74704030e90e9f9283a2f29c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463606.79/warc/dbad538463f493bceb6c219c528e37876f7bc120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463608.25/warc/37a7235fe72b77935903dff09972273012f21240.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463608.25/warc/b0bc695f474ae54ff17a42a6badfe98250403458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463608.25/warc/b87c376ccbd45a6ebbc4ce8f2de5a21440a9b527.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463637.80/warc/b6992cca78fa0793b5174cb0fe79b0abb5dada84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463637.80/warc/be2a6921b173abada9d1d3db024efd19e841c99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463658.66/warc/a60d57492f6716ff917fa32d17ea4230778b9de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463658.66/warc/e72a6d674a6540574d6af67ae0700fe63c282e4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463660.11/warc/2738ab0cd0bf9cc7c535bfc49bb2f98a1691f1ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463660.11/warc/7953f5edc54f84a1b795714e49b4b1362c9d2193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463676.59/warc/118b4f5fd8dcdf954ad32e71be78ca0e5223b78d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463676.59/warc/c99e8cbc3276b7496db2b0a911ccd7014a3b7449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463676.59/warc/f521bd22f1ae2047466800d206396bc849d30781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463679.20/warc/274be3b9d1466c776b15a1372bcbbc390dfc659d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463679.20/warc/78681bc672f97c6b1602e0ba5c987e1d387b1d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463708.99/warc/30873d99e2870c066f7eff7a1fb36891f9124dfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463708.99/warc/536fd2435ba003f701bdc00a98fee97520276b6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463708.99/warc/77324c4c3d999caa9feccc1bd74de61b118c526c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463928.32/warc/00144b2da0c7bc6f65945bbbb357f98e745da1bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463928.32/warc/4bf3e7d708e29263ef2a812714086639de2a03aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463956.95/warc/5862cfdcf7498eb1f486345dffd3715e403199a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936463956.95/warc/ac4614c0044751f3ed7ee834ae80d21840ccd6a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464088.46/warc/0c4f1b43d9c17c86e8233b0b9a73d609fba04dee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464088.46/warc/3af5ae8d8424dcab52af0ddc105bea298d2549ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464088.46/warc/8f741c6c7d239a5eed408738b90291bcf05278fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464123.82/warc/11e79d7965ad921f6c8599087149084b8bd5e2c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464123.82/warc/68c4ffe7cdaa92bd272d9d64768a76f3f8075586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464193.61/warc/77be26d4852c0b8b05747f8dcc8769e810d54c88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464193.61/warc/8a4e4cd6bd2731bc3f1eee17f3c9cc32c01ded69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464303.77/warc/036091895325c1da935111b3224b7baeefa3a7db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464303.77/warc/86672751fb33665de6691c10cbae30d7d4271aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464303.77/warc/ecdad5e1270fb5f8aff1601cf2bf33a7889b718c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464809.62/warc/b8dca6e433b12938bbd3d4f3d2e0c8a690bd7c71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464809.62/warc/e281644724ab7cf6423e7d947a7975b835a9ce85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464840.47/warc/a52aad5e449317a038ed2884ddb72c9931482b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464840.47/warc/c71550020b6388fb2685e29780bb40cbb8c2f362.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464876.43/warc/64b05ee5a1fbfb0ef2e59c5833878a118f4ba44e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936464876.43/warc/dee72a889b0baab86df564f7bad8d3f266eef405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465069.3/warc/68d3d51ae5ee1fa6f2a91041b925da294f694a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465069.3/warc/7a67ed3c745b8f2ef83e5277dd20f6e4807eabd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465069.3/warc/939ff3ff6cb4c6594ebe025ad43d1763b92ac77a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465456.40/warc/420b559450d4ae7b071110e006fc8adef658ef4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465456.40/warc/aa5a619339be4191eafb15067db93a49a911f816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465487.60/warc/54730d29d062a0aa280472628a15a0e4f2aee35b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465487.60/warc/7dd339394936b69a19b1f454868d453557f549a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465487.60/warc/d47c0177a93b2277a5e1cddd31eb9db6ea6cec26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465599.34/warc/4b54d0b8d4814a4f2d0e0dec1c797ff336c6a8f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465599.34/warc/eb9a244b687251890d4124e7bd14a921bc8f5e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465693.74/warc/792b5bc3521901cc5ce2023e0337861fda65b312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936465693.74/warc/e742e502c25e7fb40026deb9c6a182a6efc52e3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936466999.23/warc/abe7e1d38f679531d7d82f954f458cfa6f40af56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936466999.23/warc/d1806282d94a3d4fd2bd625136ec9f79862a8c9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936466999.23/warc/f13b38a6189ecf028cdb94bebc823e6ad4d907a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936468536.58/warc/c00b102b51a5f6df403a92871ca374be2069e827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936468536.58/warc/ce439eddabc5a854f5d7584e4e348548be1106ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936468546.71/warc/0c7a2c1c71809377963eafd8bf95340f67a9bc58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936468546.71/warc/e0cc3460d8020578eda272f33378693c1e6af78a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469016.33/warc/a36991130991cb2ced3c98a83ca13c829df84716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469016.33/warc/cf3190575fefefb1a6891d771d7d77fe7b88b352.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469016.33/warc/d8ea49c950a7b95b4402df611c025a5a48af587f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469077.97/warc/566d2623ba31c1e88fc4eb397e8567b524f97288.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469077.97/warc/efdf63fcce611327e79827251d4c586e47598a42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469305.48/warc/5fec88c315bfafb1dcc045bdbf609c6811299000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469305.48/warc/c1b53df30e8dea034623c7d2d9b853db48295ec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936469305.48/warc/efc32fd3c3263354657e8d97ef76b77c6e848c22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936470419.73/warc/85766fb022e9739382e1c4b3a7cbd4e2e821f5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936470419.73/warc/d397dd9c8e56c66686869d947b240ec0515f7df0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936471203.9/warc/7a0ae1898591c8b62bb7519bd7fbd7b8acf3c152.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936471203.9/warc/b30191fff83ae622d1cf59eb6ffcfdafa5152aaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936535306.37/warc/418555cf0bad7d825b53cb9c6f3bab264c46195c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936535306.37/warc/6703b5326ac6cbae0cfcaf76527744ed08816496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424936535306.37/warc/b8227e3633aaf9c968cf7bd35c02c538d041455e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937406179.50/warc/73c4d83b498f6d85b17d32d9cd1b92f7087220c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937406179.50/warc/eec23cbfb1051ab340be44e342cbce8f10717fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937481488.49/warc/5ce89eb20760553427ed41bd10e3fcb31eeab6df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937481488.49/warc/7eff2cfd1961023857193939b7ff647f41403f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937481488.49/warc/c3ec385aca2581c7d7b2a5a3f10955f297d3ca57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1424937481488.49/warc/f848de25e3857b0c34b0b7fecf07bf5d86a010b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1484d5acb306a7a3e41344466f1e33db37cb0180.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1733a42be3e5c3b17cc275e6ec87a8b1cb5c4473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/1e52d5f604ac5dc474f58459f93aba029c0eb309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/2490010ebdda0def09449adc40b9c059ca1978b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/2dab6f794648bc137dd6db3389e33605e4ad49b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/2ea3881e6a7d47f4c23af6547ebdda62279bf3dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/2ebf8867ab8c6e99d5a390e839628ce225db4fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/2f6f9221c9ef077ecb075eb45eb2c85e328f3724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/32c7f774efa693a74645931135ad57fb837bf08b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/33f79b6b36a781984808fb525369c0db0afcfee5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/353464708466879f5221dd82785c9d973ce0daf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/36c94e6a650df654aac2a09732718a279875ff2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/3743824633fb2d953b69f8ba8e36f56e670f3ad3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/3b5ef96684f573e31cedf1333fc8d77421d29443.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/3c4f8fa0d3b953ecdc83cb8c54477a8a6d6c39d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/3db421f58626adfeb369db683808f05d1a5dd544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/4492d07527ff819a1242408cda7968f328c57b43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/451f98ae7b08c7774409e5f97f5ffa026aa41809.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/468f62d4b5cde760315d20883bbbf7bb12dea621.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/4818d16ec7d59cc4f63dff97db2230d774a627ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/48e5cfad77fc28ef994f308c0d74803c8b95061a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/4dc4d192d9413d47e16dbc395a4c54c78413086f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/54591b38e0d0b9e2c41d0076b6cd498201622439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/557e00e19f79b440463ed1015af4e3c519b896ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/565995c0dc0c4ce1bb83690c6d4101642471d19f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/56ac782f1f2adac94a8a3c743192e24c94bc4891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/5a69c605f86bf25b34e82007139924c48e524518.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/5c1be8042cf8220984f0123d99bf67a298bcc67e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/5cc81e994da63948f3f28d4f7c31b71cba0422ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/602faf76cfd190b857d2f7bdbdc40c6d24d1f9c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6703f6c468a1f502f42cd6edf965bb060cf71210.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/674219cc86b959c8fd8baf4fddeab2a9f26b5bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6977e2d188330d2a58d36a01c214230b5047671a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/69c25539737c048d78745fd3b59a3e5ac456374f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6d0904adff9aa0248641be23ff69ec4681257a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6f41c51325d8d9c5e232e980dff8adbfb24091e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6f7ed89fabd22a7846cc5bc9f6dafe7a46ebbe40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/6fc2c11430f9401c37317557bb87ed909ba6fa3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7060a6d73c38d75eb744d262f33c108f88803695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/75160be968c39f58b375d8881d8bf145af09a4a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/77075ffa29887c8dc6129b574d1932538ccb8326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/776dc9024181adf052a026659372bb2614c22bee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/78dd9ebafa48134cb0fb633028113cf3f0b5657f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7a0dc656a75c3432ddbd9dcd6a63d8f16b88d110.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7afcffe881ab4d0f11fdbfbc364f9e8231bfb0f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7beebbedc452e0f196955e6358c596856ab254bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7c999d343ab073aac1ee6fbaf61df1ff0ef675c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7cb6da2ce060256ba8bce1b3c94c35cd73d8cd7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/7dbcdd2caebc7b7d054d2bf31c9ae7f24935661f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/806efff623848f5451174cf1af322bd94b81d485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/813a702ea86ab5ed29a427c04b8cf49c6294dfb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/821e9cca2735a2a4b2b37911b507e1e2cf6fbf3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/8f8b647598fd96a36c8991fadb3f8df374b70b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/8fbabf4452141a5e77cae69bac858e42feecf26f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/8fc25dd390d0ab2af6c864222a173695f804546b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/97ec298f754d752ca329f7d67dee3293a6ebf40b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/9930ba80b193d80af4c804a7eba86d11d3a160b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/9aed8c461b607defdd26dea7efa942b8fc6cb59f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/9e0781617c293e9d12edac54d71440364f94b8aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/a0f2f88eceaa4be6ae7acfaf8ef644085493f216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/a2cd35410045bed7707feee7f9a29358e2584bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/a5d7f2e8d763d76c5c6f7e5e89bcbe1f308f28b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/a67c0b9981372111ffa0f24410a6ae662f9955bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/a6f73149fe42e273c2f306c470fde24f53a936cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/aa0b7812d30277cd2f48fcf0d7403eb94f6c15fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/acf6ed2194ee92f40eb03d3b6385630c2c4ded98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/ae6ac7735d7d4a08c0484bdefe4d96b2c57b1912.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/c3e94e9552915232e2368f09eb97e9b8269365e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/c45eb98e3dfd12aac597e798920d74426567abf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/c59d0d62033dae31133e806d44540fd18e90f577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/cbc3773e18a409a0f81130d47a9d5ee737e74e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/cd5f452d94047201ea6e129e08d1cee0f706b69c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/d41bf464d81786c4db2461348e0ba685cd62913e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/d6e4d53810e6b41aa6b581d7edc186114e2f347a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/d92924487fc9b14f0e81c81fa4a47cef73e96caa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/db4cec80d48b123a3240c4e206e06fd046f06219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/db91bf3f475ebb9250a61109ce4f11d60c992c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/ddf3aaa22db732fa505ab16a8acdf783eb2ad322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e1afaa56e7f13c0bb39e7e9e201a01fcc4d69d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e1e6b3d1aec82894c38ac075ac17955f04aaaff7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e3d7893835dd751d1098cc7da199d40a2bfa8d63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e4fa4a89767acde4e80cfaf0a94b54e8d8e44b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e7ad059a990f348c3a7ecd5a3d7de3d8dc5a65c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/e7cd11f2da8ba03949a0bb7f389037a2da3598a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/ed574c7b88509c852d660783e47b4ff02286017b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/ed72635ac32138eede92cf9850da3a13fa531bc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/edd0192048fe791f9121b1f3062cb9ad257e284b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/ef54e41085dbb7b0e254ad839cd700db38fb1bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/f019a4c4105206f7dd464bb552a48954a10ef207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/f262ec11d89e6f354740a0aed5dc87ad4cede42f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/f28354e140b363ce7c3762743fb97f271dbaa887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/f376be0299d7ae030076cb66521f6a2d1d02b2f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/f8b8efd2169565001b7cdbd760a0a17399e29452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/fe80673a8823ca18eb83273393d0ce429adc2d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/fff0b11da4ae3e204ae44a5e893c8db22aa46324.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-11 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-11/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-11/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2015-14.yaml b/configs/cccc/mixer/CC-MAIN-2015-14.yaml new file mode 100644 index 00000000..23f8706d --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2015-14.yaml @@ -0,0 +1,401 @@ + +streams: + - name: cccc-CC-MAIN-2015-14 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/028e1e27dfd6df05b1c15db79a62656cf5fc9982.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0b038ab2a431465ef55d52b5bf7da3b2cdb4a2eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0b9adbc764e7496c2b456137a7b2e0882deca598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0d0571abe91ea639c6e870059d8b968402827b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0d0971a9990ee3b9c7920f3fb5cb54d2c7f608b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0e5f7efd18c9351753091eb5be5ba6533c3fb45d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0eaab183ec416e4fe9dc941aef455c1dc1c7a0d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/0eae862dcfc3b80452c38c7279451dd85366ba51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/13df18afc722048d37b65b1b1fc8590fb8cb4eff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292567.7/warc/7b6a8b145d0476f8fd57c668b1d7b9fd030bce69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292567.7/warc/8f13b7a279342e2f688a644bb0c26b7537e4156d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292621.16/warc/6298d25923155ca42464760040e5b2e8a84b6b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292621.16/warc/eb69a1a711a0e1abefad9f147068c6dc282552d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292683.3/warc/355957638360d6dbf7d5d6082a2815d5afc095eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131292683.3/warc/6e7c8fb46244157917a6ea10e2e3011d8d8068e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131293283.10/warc/2007f504c3f03964bd8eb8473bf5e48c3597e15b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131293283.10/warc/9a268d410651850780924622e732db612a8564f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131293580.17/warc/392ad3df248ad383b257ee26c9d44e36432d6c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131293580.17/warc/646fd610f3211e35acfbe85d101e9af24cbe3545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131294307.1/warc/c9e9f70cc692ff90e288441c18d4bacb78f97560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131294307.1/warc/e5bd64ba42f16569a0505504a971c5f9ec3be64c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295084.53/warc/0759aed50b9cd34d71e3afeb412c400f09f45cfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295084.53/warc/92dde3410102b6caf127634a697a4222defebb1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295619.33/warc/941a5d251342d6730d563c445c60826663bbfe77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295619.33/warc/cb5fc875cd4eb45b9c080a8e76218fdbe7e48954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295993.24/warc/0d93b94bc77ce3b563959a163f8ce73357289101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131295993.24/warc/af3aef701c320a826a4d3e30ac3fe9feb74f08db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296169.46/warc/68a1e80023b5ce1cdbb2cfca47736d478fc6acd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296169.46/warc/f6469c41ea78e799b83c3c93c37a69b741cd9da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296383.42/warc/4ad15508c65f43a9ec7691f8da9d8711ec6658ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296383.42/warc/be82defe37921c6f77971c61209e878e72a05512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296456.82/warc/719cad13a17fd1634d1b5dd70327addfeac7ad71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296456.82/warc/e3c4469b3c8292fc91cb19710359e607eb7ed057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296462.22/warc/5c8b5b1b4ad611bb0da4a5fe6bc2a3c9ffc6c176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296587.89/warc/4dbf36a11af58308e8e50f4b26e015da5d1c418d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296587.89/warc/d70a7d9a3f6cb6ab67f6424cc5a8dd92200ef569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296603.6/warc/96d1507542aba5715f0a16601848af18838fc4c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296603.6/warc/ea1fcc9f9e8444f20e9e58a58cf3d8b414dff8e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296951.54/warc/90d4c6a3b591baea9b5cab01b13d2bdba295110a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296951.54/warc/ede53a471c9ffc349202f006db435c2c02374dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296974.26/warc/49956c514552b5ce69e5c3ca219c26043788de73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131296974.26/warc/6cff16eaed2cd02d9259c6d6c40c44c4e88fb9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297146.11/warc/0c3c9fc7ad15620e2a5114fa274c5c4fdb11da3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297146.11/warc/b45cc31053c20699c89349926eb59f32e7f3a849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297172.60/warc/13940d4361478556e458989836fbc7925229c570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297172.60/warc/fb7293186e5a863e23e242c706b07c00da94b898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297195.79/warc/28ff06e67455f40396ad5ef0c145244cc6940dcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297195.79/warc/558754dbcc6813d50e42974968dda5a64cca2802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297281.13/warc/4dcdcaa15be757afffdb94e83ec0348610f54764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297281.13/warc/53f9d593baa2e628aef0ed5b674d266b0e8b2155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297416.52/warc/354a963c8e51891aab7416866cc4ce40d1ba9c48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297416.52/warc/403089d903143ca1804a2560f7e9d327196c2fb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297429.39/warc/a550e1bec1c474c314a411f5f4ca109a0615c33b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297429.39/warc/d9b98260b748301fae32988c482ac77c243fdaea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297505.18/warc/41beccf3e69ae89fa389864799600f25dddf3590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297505.18/warc/fea325a11eef235eee27a42a2ae7a44898836f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297587.67/warc/2488ed926ee0e552a992290eb3c4e0c56e8e3aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297587.67/warc/fa55a5721b36564de97a3aa98a0cd97f2716c819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297622.30/warc/390f68e6fce1868714c0567eb2f0af78cc3c1dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297622.30/warc/74ae77771dba8689b693e676030fd73cbdc235b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297628.55/warc/aa12527b5de4b8fecf6e9831ee5bc44f7e50926e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297628.55/warc/debbd452bd39facd1afea873d86080eda4fb131e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297689.58/warc/8dd825c4a55026dfe7663c26de040db1f68632e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297689.58/warc/f13463407e7b348b42a63c5c730c02f447d3843e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297691.25/warc/3376909166b920caf2ebefaf005aceb207912c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297691.25/warc/ad4508911e00319c609a67490ce03e394e9eb232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297831.4/warc/13aa4cd6dcc44c6a55b55817efad0e8953a44b11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131297831.4/warc/715d42ba244d8aa609980044966b67a6c75c6972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298015.2/warc/5e6803aae88bf9f197e5366797ed8c2298c0f3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298015.2/warc/860a6272288d0dc8c0ca27705e2e7b44d93bfdb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298020.57/warc/60afa30a79cb27d50b56ebf7d13999260040dfb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298020.57/warc/d0690c17fc3f4024311519e49d60e025a98f49ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298080.28/warc/0e1d9ff93d6ae1dd8131275c5abbd2445b779880.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298080.28/warc/7656ba11711592730fc02a36c8dd2dd5d3911273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298177.21/warc/8f21be0efebad430b8a679db4d8b23b9dfaafd04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298177.21/warc/b60f11637dd0b9e5d626ad4e7d7cf9e4133b366a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298228.32/warc/16d1dfb5a4251adaabadca52846dda077d7ae9f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298228.32/warc/de9e7a07c9aa0127f8b99bebd901d91e274f206a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298387.35/warc/d0e8400b7162e8ee85c2c8d739042ca71c67a181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298387.35/warc/e1df724cee07bf1de9d6040b179d8340ac58d619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298424.66/warc/30ec38131fac063db3f49085eb826238880378fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298424.66/warc/512fd8c483694d3ac2c4924079c4a850dfdc71f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298464.94/warc/af7fef72bb48b36d74550d4617e36ba5004a628c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298529.91/warc/0fb294befc2c93300db2149d543525b2c4e801c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298529.91/warc/3d7b9734671ce3208a43066c975bd4c3106480bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298538.29/warc/24b1064e6b0d6ead4fbeca9fe9fb5999a1fb949b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298538.29/warc/76bdc2d94d2d6be928f3157f743d53e045f1b71b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298576.76/warc/a0a957e460751c58f365228eac9c732827019943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298576.76/warc/ae768a636fc65e0e49b92448d5d93a030aadeda7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298660.78/warc/495bd332647e9727336204663d9c49fb65f7cf08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298660.78/warc/a9094c01853a2752b40841bfcf80b5c023de9e7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298684.43/warc/7f2c72e492bce83be6dbea965299da39c9f8db41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298684.43/warc/f7a20fd64c7de8fb25fcdeeeee974bf183e5e16b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298692.48/warc/28bbf978e6cc7489c0ca3d9abf60206da37c05e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298692.48/warc/43980e6fcd4a08c331b7fbd33ab2c4801c74af35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298755.8/warc/0f8d8d41d15a660a77e6c602ac7756f0513e2f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298755.8/warc/26d8d5a8ba21304715ee3fd443401ba2005c2635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298777.65/warc/40d28526367f691fd799667d1ac0ec11ceb4dd85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298777.65/warc/9754e0ac80b7fc8cb0e7c22095e04ccc081c21f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298819.92/warc/29afb61162b67ee7af2d10f324ab62909c9221ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298819.92/warc/2e859d4085a2e69cb2008db17c82569e49e10b00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298871.15/warc/1b854e644e00e12cad74bf02287cb37262e2c7db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298871.15/warc/5d713fbeaf40e40d107be4905eb5166962be5afd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298889.70/warc/32762c856ad31b309658c1e8df99f787e7ab5573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131298889.70/warc/e773ceae0d5c4c0408b9edb4bc6829140df7ace0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299054.80/warc/38a294f5620091592944471cf468be3e42abe972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299054.80/warc/583aaa5e0a478618f991a7d7f7408a2510db6447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299092.45/warc/c582e024e3a84958f2c5d87722e92d3fa4250685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299092.45/warc/cf1a25defcddff523d8478d5ce1561e9e830b7d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299114.73/warc/123e1016c4a4a2b1deb5674c9792b3c725c0a1c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299114.73/warc/89d734c57a06ece41b20d6c13311a2c0a27d051c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299121.41/warc/c9149ce2d56222e154ac83d0d90aa55a219e75f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299121.41/warc/ca79a98b798abf8a9b26d936f8fe90691b0277e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299236.74/warc/9688c1575caec77bccb7490a639a50912119d104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299236.74/warc/b8c41fdc3af7ca8690ad78f6d658c7a524514b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299261.59/warc/9d0efd25c2744defa22e39b5dfcfce6c9fb18c29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299261.59/warc/e51fc8dc8f96759b89a115d0f7b3b3ea50457b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299339.12/warc/4d770aba7fa714f0c0ab56256840066512e2ead9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299339.12/warc/88690d9bec6783b35cd86a42a4b16100eec3abcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299360.90/warc/1065d4a54931d845fdc69bbf472efa503c94a6ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299360.90/warc/13220322d198bd543d31cad2178409b575182f68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299496.98/warc/5d99774163c9910b6c552cd9afc065d4bf590abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299496.98/warc/d12e6981f4747575c2fec785b1d5064f7c90208a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299515.96/warc/4245a59ac19434d2fc3cdfac87be9ef49c2ff09c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299515.96/warc/cb9024ebb9334644ab925f4bbea4677d63e84143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299877.5/warc/b3f1a230c812d3b40c289ab4cf1c19dc8a021250.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131299877.5/warc/c425870bd77b639d8b4bf7a7c22be1f66123bc61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300017.68/warc/84c5fb7efa9a5f319aed5361ad10f6f4e302e1cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300017.68/warc/dff86c19236c7c3a9d919d58d59accac63100af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300031.56/warc/6727384efbae40556ae2d773c0fe14537c7d96c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300031.56/warc/f2f352d25302077da79adaadd0a5081dabfb5b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300031.99/warc/f0864aefd6f81fe0d3e9d38436a8ce649b9202e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300222.27/warc/7bb4a59f5ce547c061994d29e320e6b21beedd44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300222.27/warc/e9e20e8e781e260c0a17214baf2ab70ad8c751ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300280.0/warc/24cc6150b1ef1d8fdeb27cd083f4208117e09fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300280.0/warc/9535f395677745ae2a68bc78636762c3343a8e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300313.14/warc/df3ef85a2e9eaec88b95b9038b92ea91a503649a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300313.14/warc/e08fe5678e7ccb1ae9609636910948accfd8b490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300441.51/warc/25173bebf84feb688d95433c1b212de49e830585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300441.51/warc/326054e0ffcc536e27a8a6df0907ceba4b3d54df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300444.84/warc/01aaf74a2a5868b3c941406ac788fb241b5ed3be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300444.84/warc/49fcd68cc1307d8942f7dc669eaf6636e66a6ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300464.72/warc/170611ef1b612ca856885e16bb02c1933cd61ee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300464.72/warc/74f7632bfa1f16c1a37eb0709483a3b16b8898c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300472.97/warc/0bd05a102a0067afacc757e7fd607ebbe08ba37c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300472.97/warc/5f8f448fa6e5526ac99f4a9098ad47f8183abdd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300578.50/warc/388639e8ed71d538dbc0e59180ed96bc4ebd1983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300578.50/warc/3bf09d954f6f1ea55ac6206ddc1e84ce60e30322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300735.71/warc/94fd97490ceebf7501d6113a14b552f66f53a706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300735.71/warc/ddb883319ad19b237d6b3383807b211558988b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300773.77/warc/0ec64f87139325609f65872bbe6b123a9407737d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300773.77/warc/8f679a32e8175f8e714eb33f45b56724ef4a04fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300799.9/warc/1b7c9b826eab54ac555fa3414ca5c4db576e5a77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300799.9/warc/98f5dcd1cb7e041c534b17d0620f7fa0a9c34c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300905.36/warc/1cfb10e9a0780259f349771539bc8b371f27c0c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131300905.36/warc/d148171c98060b59cc66560581ca6430bc36d4b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131301015.31/warc/3f60def855b1d12a89dd8198279cf8f76d7aaff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131301015.31/warc/78da618cacafc29fce46ae39121a84b87287cc61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302318.44/warc/0fa3da9380039a039b676c3a4a22841cc82900c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302318.44/warc/7a4d7c2f228feed2a6d1b1ecf3026ba56a060a83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302318.88/warc/4671d571dc6940d299ed410fac029233cbd34697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302318.88/warc/a65a6cd8f8275308c4c05324b9f24fb81bcbed4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302428.75/warc/96ae23a4625985a0f005c6f9e5b9b625d48eba69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302428.75/warc/ad6cd3f31ea9e0ddc50bfb36c0ecd8b578c3c55e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302478.63/warc/b6d46bd8b8a8a868f8e2e87480fc55b71d31c8a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131302478.63/warc/dcce5d8fc4e607694bca029ceec7f7b839770dff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303461.20/warc/1d19993ff4008c1ead9b275ee0d0357bbe3e4f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303461.20/warc/bf9cbd2b4820671bd465b30e08d91066ef643903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303466.87/warc/bdb4de1e56e9eb48f9707193a0b671cef7666c00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303466.87/warc/f5d6fed0495d8741c02e4af2b5e8bf12deaa2c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303502.37/warc/09367e455a4381cbfd96100a0acd5455b33a24bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303502.37/warc/2c36f68d0d59ff7e67212760361e2965ac79e1fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303523.19/warc/1207065b1c13484b40f9adf760c49dea519cb020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131303523.19/warc/c7c95fe7961e0d3acffff19b6d3543586b10ae0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304412.34/warc/36314233ffb685759ce963abbd16a889d0717e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304412.34/warc/79e938e31b84040ae096494ef08eccc9923cefb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304444.86/warc/9c474dfc21fcb37e672d088cef0f912604e75432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304444.86/warc/f5018de745be30c980622054a7921e44268a2a33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304598.61/warc/22d4c5c64b34fa88e27f36fff93104b52da06782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304598.61/warc/f1eb1ca682662ac309688bc899bece78f7a9fc0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131304625.62/warc/9b914a75b5836039804079f56f6f916d8396b60a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305143.64/warc/4eb7bacdfd9b89231338c400a6493f894c659e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305143.64/warc/7345ec00a4e137610ee6ae1d41d3ecd99b5a999e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305143.93/warc/438451befa83616b6e078409ee2c928d00c53a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305143.93/warc/7a5aa18f5ee22e79e42f3cf10c3c02c25f4b3741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305388.23/warc/20ab95763ac9de777372ac05c7275990ba7e2298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305388.23/warc/35fea9f79fd8e86daa6e996c60b995565d3076ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305484.69/warc/dc6607b5b9fe541d0c32aeccc2ab3db1a78134e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131305484.69/warc/ee8fdd51d7247c4f2e7ff34f507e6df2da23823c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309963.95/warc/a02653d06ed2ff77fe2137d5902e523e52d560b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309963.95/warc/f59bac246b0bead5f4abaf6723551c0608b21bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309986.40/warc/1790fe0f243e6146d833507be0bebac2da916cfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309986.40/warc/26eb9431975ecb5c4f0dff1120c568d5c386a121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309986.49/warc/8f0c9f692278bef1322d0efc7d39bb409c1eb54a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131309986.49/warc/fae99df4e69be967ccea763fed3df4fce9b57c4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131310006.38/warc/716f003dcbf2b718527738181702d553fce2f2a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131310006.38/warc/a5cf41d191612064f9a1d18ed8b6fd11465e8c57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131317541.81/warc/b8a6aa08afa5c9780b08874bb12b6a4cec071421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131317541.81/warc/f34ea77c0eac0b0c2f2fdd09d20ae644cd13fbbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131317570.85/warc/b39ce78005aa887cd73bf2ecf9e78a58e686bd96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427131317570.85/warc/d12a9031c44749f0472a3666f6fbd359045a817d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427132518903.47/warc/26610699e1b7e0f0dab9ce42b1072011e507e0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427132518903.47/warc/99ed0cc361271e0509c0f94f63673fb040d6e8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427132827069.83/warc/24c4934b4b3f115b48caf2de8c11d1c4f5119416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1427132827069.83/warc/9ecbfe79e52ce7aefe57f10fd6a77ac76f643641.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1c56da2dc6cf49c19c041095e572e0c34076e618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1f337d8b2b0ace678d584e4d4c9ddc77577395e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/1f50d135d0d3229834f58d4090fb04c7383d2470.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/283608c77bb89603d143d63725913172c752efa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/2855dfec773cf237445ed46bafaaa196b287bfd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/2c3ae72efdcc7f4afc68e416461b005fc20d542e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/2fdeeef5c9466bb94736323ddc884001ebd8e456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/32e365935ac31aa8dbcec857df8e931bd848c286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/3d3dc34a79de1a62f6510e01c1b962cc9221c790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/407c7c3af7fc3a91166e59191ee6d709250a8318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/433e77bc6bd8e911136d01ece9565465dfcf1bea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/467cb62e28663a0af8b2af83eef0564a649aa60b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/507cbdee729a199eac6af459b77e4a2a422f6ab9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/520eb6425687b50df12da5d047265071cc149d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/52309613ba6a56b4545a309c3071df7511eb6522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/5f279d7f4ee6c027e427c01db38c72b91c0fd994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/62d3843b5ce714e041335fc2789695c1b45736eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/679dd784fd1613c42de5a607430738881b3f36fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/681acd9d58a88d7fdc15a69c5660de5b4d600bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/6b22003392a63627110a9e748790225ca80a0f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/6b8dca6ea9f50ae8601a6c2af7fc0761ff61a371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/71c11fdc3bec393868fa72bb55c3f1d758a58665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/71e3173039502aaaca61006cf16de6a2b042e811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/75550cb605d28b2f98c3493dc90d07e063b6c46e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/773c5075c5b8a457bcd8c1a1f0019e86295f8f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/784bb9626a76061ae554b35305dcc6ffb78c1822.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/797b07437601c0b2b0b7a08c4fcf769800b083b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/7d73615c4d0af72e55fd0d4218279a23a6f78000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/7e31a12267e75edf07346cdc153f8dda0d1c604a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/7f87553e42957e9310231035578db8b81f645f9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/805d91d6dbd64b3319c7a15c9cf696779feb83c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/82dc999418dea1477d2390860d2bd4385ad5dcb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/839bf027e320b60a1d08e030181286be5a15bc4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/852717d96238e6c8b82551c1ed67d04d974d6555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/8747b5ca0613725018c739850291e805ea4a941e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/87ee34d40f0e640e5828da031af796a0cc446801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/8abfddde17d121edc0200735e5de8ede16472e27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/8e3c42208b0e31ed2f75453e59c3d9f18ac9058f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/8f2e7bdbeb3ae1bba3aa2a9eea192cb9ee17d8d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/94f46f183766715a4d635da784deb65a74da179d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/95753dd5ab53fa0a972964e4ddb68b4eee8ccdbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/95fe5b72e594ca55bf788117d8ae8b7e3a314e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/960dbb090af618cbcdea43bfaad38a801f64108f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/973d6d8f1d852b87903e887bce8f2070bfdfb21b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/98b7565df5be2eba9df617a654b56c391f308adc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/9d1e9572d1f9ba3f5cb6908213fb197bb0de6c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/a1b575ba501ce5c03a1b13998f9dd806355fbb84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/a527e7b85e76ac3e42d2f29f956050b34a0fa2be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/a59439b90c1bd977246a65b764f4407a69ec5445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/abb54d6235179f10bce8c2178fda0f7345cfba08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/ae24fc3f338460088d9e79c18e3c5dbd52e3e650.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/b36039baf4f523a0fa886777ba9c374f84a85aed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/b3b65ae6adf269498f120dd18118afaf14d73de0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/b75dc60a02a6428038556aa2b1b3c34136e82560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/b9467af637f7c6a52da0335bb533d763ae18172a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/ba7227955cea66639064d480942bb7398b5295d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/bbe00948d7a747d8a19ded8bdb95d4ba9d327e00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/bce72c73eee2295c4f5665f916cc6b99ac158df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/bcf977f6a407d88b628be6bc7bc15f70bac095d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/c0e493eb71adcf0c0c13930e17d81b7aaa308c83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/c66eebcacc9221ad651324f8cfdcef9c77e2926c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/c864ee120106a9e43eddbb36815fe4968092efaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/cc0aa39dceb146d1dc186a275dcf76d3a6d9f932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/ce14f954cd0343002d02f4cfd56416e9e4f73f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/ce438305e2f4e88abb6efacc346abd6f2ccc5579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/d04c14fa5e26122d6816292429444e44e13848ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/d0f0658e6a40a0096932579442b7e9373d582226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/d2fe25e5f48762e1d6199759aeb9865f28fda08e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/d472f8bad25b26b76b95ae8be38c23db851ffdfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/db607c74412e584d3e8a45b7426afea2a04e3996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/e2bf5fd68541d52b7f1fd085808d3d4e71fe5fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/e5870943f84954778948cf87146df32d901d3a85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/eb6cda3b422dc5f975915321e1ec700b16318737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/ecac72942c623b4294f65e2ac8f5e5a5fcf47740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/edc0a03775a6de39af156ba5f667c3e3ed0e56a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f1439b491ad6320a1a2ac4cc1938a61a8f96e26b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f1824327f1008f19c80bbad8411cb592b23c6a06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f18c669a69267dfa74d97874e96f3f5300fcf206.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f3e488b1e0983544ac7160add57bb409255f83ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f5af9b81d69c0313540f2960fb4df94c6b3e9f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f6369632b7c04fa10a4053f1a8523abaa839fd1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f6840110cbd6df1bf1b8984f34e4d939083d3e94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/f80cd8fb68af9d14023602402a138b06dc050ae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/fb43a23af497a502f5bbdcf776f335013b1332c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/fd1936254d98e588202abecdd4a3aa3d78730458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/fd93d40acb2475a50f3313a97de06551adf12cc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/fd96a741b34e4627683c8d0961e5f3b71df6b290.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-14 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-14/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-14/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2015-18.yaml b/configs/cccc/mixer/CC-MAIN-2015-18.yaml new file mode 100644 index 00000000..04fc63b7 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2015-18.yaml @@ -0,0 +1,496 @@ + +streams: + - name: cccc-CC-MAIN-2015-18 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/0111139cf1b363ac4c4a7ad8c6ebfb65ca581af8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/02d7ec60b68e2c542f2da61ca2d40869b6e072cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/0324e49782e208cd45632d7261fa8f7f8c7cd8e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/035460fa1c4bfec9aeeaa2123cc286ab95fa4e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/054af6e715064323cfd4807a973a583199e34796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/05c5fa2f00e50e56904d5be18aec26ea9adf5c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/066a195bcc79b70d824bbee3125594cee6ff0f27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/08ce5504296ae1879fa08f659aed2beb2ae2a941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/0dac07a56582f25a68170e9d9378432b61b02379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/0e5beb50418201e653d8ab99d7a559411a2a6eb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/10aacca0ddae537d3ea2d40151bf65bef42dd88e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1372416cc9998668813200b4806e244ad6e4ab07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633512.41/warc/602cc887dcbd38af83155d50d928c62f110990c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633512.41/warc/e40e15d076589c4813d378e50c9c63eb3b66d814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633512.41/warc/fc120f1453747d8c8fa56d57b045f628a51a9394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633799.48/warc/5b80a560b5605df7f93023060cfcf68a46537db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633799.48/warc/f2f6cb08fa0273d1ee9bd07453d6761cdadcc8d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633972.52/warc/85bfd7d605bf0cb01710aa8b81e6fb496815ae2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246633972.52/warc/a72fd57c86b5c5e3cd07e163b5344ebb44fe1d8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634257.45/warc/5b03f7384535fca6691a892d0ea5d006a8af3f49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634257.45/warc/aedcbddda0c1176f31cdb8f4c6cb9f17ccd5d953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634331.38/warc/19675a5721d1b33eda9f41dbb403b80db726808e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634331.38/warc/9e7d90f3fafa4b3574a753df3c0b4282f7b94671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634333.17/warc/216e2740a23f493dffbb524a987327ef79ce791b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246634333.17/warc/36f3b5d8ce8f99725c95200d81757f13d8fe49d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246635547.24/warc/8c99bdffef88f0d760bf1f8623077bb970dc6535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246635547.24/warc/9fd28f9737d1b4c9d5ed108faf0cd538ea0e63b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246635639.11/warc/3021e5ca74413ae27351988a7fee5c95abddd86a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246635639.11/warc/306978e587024d40cb5ef6f206e42e7d6013d5e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246635639.11/warc/e3a7e676951fd90a2917980090e9a30a77623893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636104.0/warc/a1211d03cc029d01d6a56ec27c91cdf7a314018f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636104.0/warc/eba9000109522110dea9271679ab06b253e3d5db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636213.71/warc/148be045105692a99dd714d25c21a411fcd4e9ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636213.71/warc/98fe467123066f75d405faada5d53d81c7409c04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636255.43/warc/66e48ebc447d63deb5c6882491c3979b06dae521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636255.43/warc/f207df33ba2b0f584b10caa2720fb764de0bd603.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636650.1/warc/17fc4c3c5453a212e28ed53e643a4c3b5a160d8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246636650.1/warc/ac91b59c86318c5f0f8c40304aaf3923c0e79b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637364.20/warc/108599585b1e377a56cfbf65587a2e4783c4584b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637364.20/warc/273e110ac1df3f4f77f4714f54a0b8272c8f0e0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637445.19/warc/50a6f3c5eb69a8e24836948aed55b4557d1df6b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637445.19/warc/89a296896e40a9e4f6e07249551976a59c73e9d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637544.68/warc/47381300498a3ce436bf1290bf98a60b393ca2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637544.68/warc/5aaa6d5ed2455a6598789e4722ce99b0aff1e17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637979.29/warc/11abbdd9b1da37a3539b8e3322e5fe5eb5f8669b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637979.29/warc/23743a036e1b5f98ad8d3283b7fe4da2119e86fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246637979.29/warc/445f5e8df42192d8333e6ed3ab4532394089e9ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246638571.67/warc/1f35973f07403131c0b043b6c381f8835d59948e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246638571.67/warc/651fd5006053d96258e33678abbed2675294afe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246638820.85/warc/52e63d1352718afad388af6acc6b951bc015ac6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246638820.85/warc/857cd79b4e2bdf6ebf14d8e41010572c8a95750c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639057.4/warc/e0b92d33799dff8ff37bb5f07aff497e841b5fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639057.4/warc/f6a8f4e3c314267f1cbcd1d359a61ca17582ed5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639121.73/warc/1eff24c453c818301856bfb28c1f8465734b1943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639121.73/warc/b53407e0809f3387223930726902d798f3ebfd1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639191.8/warc/4b676da15f7a52b69182c4d456d051124d73f108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639191.8/warc/5a3d4ce80ca6a1d3450e187de2775b99a8179d3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639325.91/warc/7c615d50a5e6eaf3f6cb9899d332cdff04d9efde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639325.91/warc/a5d9e4b2a27b77a14546a73af531f44da2c99f99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639414.6/warc/22a4e79c590f4a32dd6f3f5d28c6e754135690dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639414.6/warc/e2a78024f777d0803e11c40f4aa36814d7023b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639482.79/warc/3d2bb8355345e572acaf118092fca3a9adee53cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639482.79/warc/b2a2c155a25dc1bd4ae8653d24d2d0d4ff10ae8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639482.79/warc/fd51bcbe440321b03be88311961eeffc9f4fe5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639674.12/warc/0a1ea0ee36ebce2614896475e4566f8c2f6baf01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246639674.12/warc/935d243afaea6235699ad7a8c6e224257066e883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640001.64/warc/2372a1c8335f380eaf0a7e5894bde4d5835e95b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640001.64/warc/6b798b03114081eb7838ced1ffd899bdc3ae51ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640124.22/warc/874f8ba9351ea69218c088d94d273940698a7889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640124.22/warc/b39ba37545b1fe6fa6a16bf94fc3ef4454eacab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640550.27/warc/8ea32c28b00b9ef56aa9a95410c4576b98b38d2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246640550.27/warc/edcd659229345158fc1940f4752e6463e84960bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641054.14/warc/18ef8ca82177c3d280bcf193ae60ea4cdf9af308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641054.14/warc/403a12bbec8757a293911ab873fc51c88784a870.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641266.56/warc/208afecce64b3e7177d539896e1c975671763bdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641266.56/warc/49c1715dc86ddf4377781838de26655ab12fdb2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641393.36/warc/15fe16f8acc77de61f0a05aa5650bf9dc84795e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641393.36/warc/e0037d8bb42effb301a2e6aacefebaa9da9884f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641393.36/warc/fd5570c93ff874f06891e2375b4a271dec66eb7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641468.77/warc/40986e755483ee65944b054a0501ef9f63977f31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246641468.77/warc/911d7f69de81087a248e854047e96a5db5d09eb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642012.28/warc/30f00640759bdb546658e99c1dd8deb6e94f03c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642012.28/warc/a12fcb970243914f48071b0abefc13df4a723f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642037.57/warc/25708fd6dfcb965641fcfb0ef4e0cfc65ccce6f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642037.57/warc/7bb5d21069f95bd6dde09875b604742b572af1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642783.89/warc/a87f8add4f129ef31148f485f9c530603b0173f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246642783.89/warc/f95258a30ba01f2934ba6424766a3bc421f1637b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643088.10/warc/8eac5b1e1365c648b8a65c1f818cd17d68852f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643088.10/warc/ed0af9862ed044eefd0e1ad4731bba5a254aadf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643283.35/warc/212c0d56531f97bf7f85f9fae0948add281021f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643283.35/warc/469622edec6dc98e7af2c039ac5bf2c58bf516c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643815.32/warc/3c22b85300074e8ea293d33a3772b21fcbe4d3f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246643815.32/warc/5fafc0e64313778542acdebb7bb13e0c0f7d725d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644083.49/warc/2b5f3663a4bd6e2a7fdfcdae5a2cc0d867e1057c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644083.49/warc/dfa619c031ee2fa3b091b73c626351c86d88bfe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644083.49/warc/ebd1ac2746c3821937d0f028fd94284a9cee2c2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644200.21/warc/3d6b26cbd60cc66beefe4061294b2e0e3e94b48a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644200.21/warc/486696d342bd1420f95633c10562e1fd2c6552ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644526.54/warc/d2e132314654a9fbafe3e7638e81c8f65d24bf89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644526.54/warc/e0e891bb31fd03e2d9a968e8885e348410b5f493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644715.25/warc/8bf52836464754069e60457797b1474a7fe426ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246644715.25/warc/9a2e9d1cb9d1285da6a970cfed7bc2b1c93e6a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246645538.5/warc/3a722531adaa75125ffef0faff838023d5b245c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246645538.5/warc/ebc16decf4faa2ddc2308fee62fc144f114f04d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246645606.86/warc/4f27a80b13ebb47085fdd782de0bf7a5c32e7ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246645606.86/warc/6ae71bdce797331454d743114f8885739b217fe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246646036.55/warc/3a2dc45fb109a2ca62be98bc7098a103041b195b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246646036.55/warc/fa5674c77548880a75a2f96707abdece5bbebc63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246647589.15/warc/0394864941f8dedf1a12668a1ed28f62070f485a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246647589.15/warc/60613427b7158cebd9a00b03c97682da0e8297e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246648209.18/warc/17bb4f07cb874db13f9cfca66db1f4cbf5083d4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246648209.18/warc/43fc163d566ee6377ac67aa1c98426d3d22b1fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246648209.18/warc/a1e897497967f4e6fa01b4fc0c70e58b3fe7ea6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246648338.72/warc/8c82999372c5f35c9c48efe798642a87da203a45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246648338.72/warc/97ff64dbdf5638676a19870a42cd989504d07311.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246649234.62/warc/696d5ce699f974152ebe38511ad6e54808db87ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246649234.62/warc/f248babb5d5f7294d8c4935f053f9f4a4ab89db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246649738.26/warc/e92deadcdd8f5e17744b44f19bbefcc44880b095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246649738.26/warc/ee01b49e348804bdb891f554d15e0fa1461b48a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246650195.9/warc/d01b8ddcedc17db57f18090bd1986339d491cc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246650195.9/warc/e2ca8108b28ccbac89a60c07d019206cab0ba7ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246650671.76/warc/3b8796c5d823f7837426eac4e11ca968454c2645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246650671.76/warc/9bb21b24c2c4a293b23d3dcfec2e6f8646c7def2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651471.95/warc/2dbbd8116ec62b8cb69fa714a9eef820a4b124dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651471.95/warc/5cafc5f429b82e5e55fb16353f44624123cae008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651727.46/warc/0a81c99bcaef57ead592e0f82ce4fcb6cd6fb54f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651727.46/warc/d9473ec79f41aad36f67ad8f84d891d5d9e1f619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651727.46/warc/e2c619e07b35c668d1c8bd50fbe00f13cfa1277e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651873.94/warc/38e6662648a9429ca45e201f4cd75da944c62a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246651873.94/warc/c41043ff60a4ec6cb587e751e6be6123d79c2d18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652114.13/warc/4c0b6a49d1924ace05284d5d9e1bb97542d9194e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652114.13/warc/7635dbc33c4846ab5659735e11d19258ca1cb536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652296.40/warc/34accb451a0388159b23b12cc2a96511775f7b43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652296.40/warc/5d888aa7b9d928adecf201085856ea24e1107c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652631.96/warc/2b1b9d6b11370f8162c8b46b293558c24c01cd38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246652631.96/warc/9b88bd195330931111ba9a4daa0a04d175086795.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246653426.7/warc/b26755b13dc8e040e19150f3477321df141ce96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246653426.7/warc/d2be67b43749e5c645fc64df8b0d8b4fba450b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654114.44/warc/af2999bb7e91a8fec0b2e6cfffeda53336a75896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654114.44/warc/d0e8c1705f29ed79e04606c9c2afd0afcc254d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654264.98/warc/7ee23f5d77abcbe769fbc3c051f41a42ebeb9329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654264.98/warc/d59927115adaacfdb78afa5db8eb503c845f72dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654292.99/warc/182ed59bd88b05f60f82063c86888a1675b03f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654292.99/warc/3680483e5a74e6e9abc4b33ab700792b03992b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654292.99/warc/891777525de79bb4a66199613c6f78bc2764ede1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654467.42/warc/09bee76a8edefd055aa0595d8d50385433ee07d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654467.42/warc/31650c45ceb3456b7a97b8567c072056dcbbd634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654687.23/warc/43a40d8bb182e2d3ae4cddcc5cd820fd55a20b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246654687.23/warc/8efecce6058db2444cec09525bbb3bf9d8feb9da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246655589.82/warc/0f937bf4cec654394a38fee0ca49bec80099359c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246655589.82/warc/ac8aa2a7498a69ba97f253bec215d72e6f022289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246655962.81/warc/1dd2a24200f55edeadac353e066ec0c484f1e943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246655962.81/warc/62a10ec1d757726343567c3b725616401e5fa47f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656168.61/warc/17f363a73fc8f8c0191ad034a14bf831ca46be26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656168.61/warc/f03d725ee833cd4e4ff7e8417a86a1d355d8fec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656747.97/warc/3a43177fa51f99bfe26731699421f8cd9b54ddd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656747.97/warc/fa5c58fc6718c78ce544386efe89bbbaa5a2e0b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656887.93/warc/0dccce2009db00ae8c71ad42c9d37d935cbd4695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656887.93/warc/3fc7198f0fd2af8cf1459dbc72587cd3eab78375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656965.63/warc/436158bfe283a7734330d3420ea5c8a93d1c65ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656965.63/warc/69de430df4452fd69f2590bb96179fbeedb47015.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246656965.63/warc/7d6815034d840c439a6f7bf268bda015d58d9313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657041.90/warc/6ee3537b74725a51787e601c876e83dd7bed2536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657041.90/warc/9ecf69a53ebd570ef76f7af0d6a63e23c56cef7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657216.31/warc/7ad69b8be17984381eca67f714527804bfe59c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657216.31/warc/824d85bcfa41f4f783dee60432169a3ee25b2d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657588.53/warc/d059b0414c77e9bdffd42a017f6d32f67cf6189a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657588.53/warc/df3cf38d9f9d53cb08d36b0504300ecdcf27712e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657868.2/warc/65a10a3f8277b41765c48ab8f261e4f7c966dd9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246657868.2/warc/b2000e521a554a2348902ca543d7dfe79fc05eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658061.59/warc/ad0a6c5f5dfd85fed2a9d3badb427f5658c1de84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658061.59/warc/e921edc017cb7a22056f69eee99f1f06451724f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658116.80/warc/b165309ce97a9fd9c0f06f392ba932e83020b75a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658116.80/warc/de14174ad6cd0505fff763fe67dfa325dd5e3e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658376.88/warc/60fcf67870d122ef4a1d384b58d4883c789d45a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658376.88/warc/878e7902d8614efa827a73f4d6d8bb8d202005d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658376.88/warc/d73350d8d9ad5ddc72737900a6a7f7f3079ee4c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658904.34/warc/6c656f4def4b1778b3e24097a528809c57cf1898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246658904.34/warc/d03f5fa19907a46751c3c5deccde3fb7ed540b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659254.83/warc/1c67aa8bbc199e0d0fb276801be0abffb5b5459c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659254.83/warc/3b95a7363a313d4d247b4c8e71b08a8cc6eab69d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659319.74/warc/014514eeae6a4f67301e4706705ba24cb9940d38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659319.74/warc/98ae95a77b099f0874cfc459a518928685996204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659449.65/warc/828961f56a35f7ce87b71edb4c6208b57e82869c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659449.65/warc/f38d021a4ccb6df90ef331df8dcfa4679371c0b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659483.51/warc/9e3f50aff93cbffb7c3165c31974a987f9185cde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246659483.51/warc/a8abd92acc2b128d9dc57e97588be79434d80b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660448.47/warc/7d0a0f85b3622e9ba1b66fa07902fd54abc57963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660448.47/warc/d3f685fec3a16b1055b9992fc92c92973e9d5a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660493.3/warc/a281f257452b9928fb83faae539c8def6809948f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660493.3/warc/febad806fac6a84a6391560a31190580c21d438d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660628.16/warc/0e47a594620d5787f5db03ede27a5b702e7867c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660628.16/warc/5baff12f20ed68af505bd04d12ce39dc95127bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660628.16/warc/e48cf7f68178e4bb6dbd1355ec411a3a743fe510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660724.78/warc/0b51211096904fe39a75d1052ffdd7aaaf34e470.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660724.78/warc/675277aaa0d10b2b2639a37cd0b415ab8cc34ccf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660743.58/warc/1f0f259f8abce75f7f46abfc73c8dae45d00d99d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246660743.58/warc/cbbe1d1096f27ea0c394bf28dee0f8783f55e370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661095.66/warc/b83baf9cb83f7e5a4821e09cfbbef4f4b87d6821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661095.66/warc/bf1e1ab35490301cffcbb6b43294d051747c5e54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661364.60/warc/34b822f49d2954dcf18e53db24e42e22f1671ed2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661364.60/warc/9d3f481d9e12bc1ddff35bb1f7d0ad463b6feb51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661675.84/warc/87ab3bc59d64bd24b29877cb50a198044c549ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661675.84/warc/9ea6ea771f22a91ae1af0626489e17eb632693a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661733.69/warc/1a21b3fc9316d0c40f1a307edc91c64fe225302f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661733.69/warc/e771971e9635f25e9369817f2341931ae70924be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661916.33/warc/3a3b884cfedfdb3f28a2ce624b17b909bdf9b93f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246661916.33/warc/fded9a96fa072ae5631282a403c46eaef0d1d7a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246662032.70/warc/42ee0c6871ef0170cc46c5781fe088a16a434292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246662032.70/warc/a3a134c80cf5ae8c1e39b6aaa3c336d05331c396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1429246662032.70/warc/fa1e716b3fb5fedc47501875bf6e1462174422f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1430461119624.60/warc/e21679323df7f99847a757e9035c5ed9ebed27dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/14f096ef914c0725db2ab4bcafd8278ccc05bfea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1693ab8bb1b5c1d855c2046033ae55bcab073adb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/18040c95a01a3f2f6cd3d6aa14e1f5e7fbc642f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/18e01b70706dd0ba10b9d45020e325a12936b098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/198420c347a97b353343369abc48f4ba8ab185b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1a118fd980588144867ef555a63d80a6b22842e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1a5e93f535faf1da61a95ba16d3ef0c672cce4c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1b0bca436c6edf9b592dc86b173094307b66e855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1b60d26466bbf9ee18f02fadbe747d3acb800894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1c5f70a73296ff40c49806b5dc2403fbd6272e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1da3602ca59438a020bca3562add98bc4ca43b12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1dc2cee3884048841372845b8162b4ac2a93a7c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1de5ddf8bb5d43473c7a9adfac387aa43dd24335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1e05a0af05517cd1dc7598529c10a677ce434907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/1fdc9d549a885f6201e214d5956c409fe69550e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/22931177179dcaa926bc61b517bbe2192b87aa0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/24379c4f5e619bf82c682292d93a7deb814f3048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2522da0272651786f112e9caa85c094ea74789bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2674df9cc6a60b865edf94a5294e982dd44f529a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/26c5bc9ed3f9b1a4af2c553c2677803fd120de9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/28e78c7e6846408afd8989de61aa86459fb1c591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2a828c5d2a2a41c099ae444f39a5f03a6987ac9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2b57f253960521f2bf45802ef15d617cb1695aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2cd699c33fd444c392c64d0ffaa626e15f59b1a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/2efe10a21d66d5ff90435fe1c89281ba391bff15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/30ea57743a3d859667a7a2f1823c713670a823d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/315d10badc4318542f2c97ee1bac0cd205bf1ec6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/3184ffa55608d934a351e3ab0e8fee8463edc566.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/343d17229ef8fd91cf0f9dc2c13b1a251e4224c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/3463c5ed1ed38a1dc8cdf5f6f3b112ec843db254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/347a3b3782913a4c5d24726f84263bf68a2428cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/350f7cd2fa2b6d791b374305c73cb6478e592a45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/35254f0abc616b0553e908442001e68e0e8a350a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/35df306b8ba939ce54886e8468e0d87a9b153761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/35e91b1f3df55d2f15e353451a00e1fedb36b2d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/36726fe106a692ffd417edf8fa65d34193e5f4e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/37c5852c90d796af70c078e1c3980bca631a695e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/3dc96ed37465634f100c4b6cd09b7c8430a0f77a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/40ccd5fe8a692708e4546a6c070a3c6f477dea01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/42bf78e5f67adcd1755329fd5c2ff003c5222e7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/43cab8bb17997f4c3bf62fe13a006e2a6acfdc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/44c6c0521bb08d836fa44438b9a87f37f627f318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4562e4795e4b8e3521ec8d3334c87ff0aca04454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/46b2da9b67bc6f10bdb52ea11fa200a8abe39e1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4b476bac29bf02e4c281a3b2a74e30a1b2637275.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4cfc478f23a7b14b2dc35da5f49190f199f030d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4d128c893329b8c9b7c8035dae39662143509be6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4d387e30203089866d6c487bfd1881570c3c6452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4d84304f69cad794d55b0888bd1a28c2be55ac21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4eeaede0bc5fa4f96f71a8a53722c0a04072656c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/4f91cfec7e20776f7df68ffbd1e76a5652ad2218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5362eaf898e020a80d248ee9e60669749fc9f98f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/536d2672f7a052ce47632427cfc94293b795edcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/573ce41a1c7d0a7d5bd85a6ab68607d850886388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/57ffc731e94693a507990592f499251d0d029f42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5aa54ff2c5f4b4c306650d087a0c90243f64c245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5b8c6627fdaf24cb9ffe93d3ba2c916854910be8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5c60e150142df1df72e32d6992335554d2d85693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5d6670d92c01422a1ce722fbb9b963a7abe6a047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/5f9aab553c31d0099362841320056a701c19ea0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/65a4b32fce14a2f1b7229de41f6f239a1308edd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/677ce27b79ce12069e2de81a606b0656e1fd0594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/67de3ff6f5dd75832b25dad3e2cd578491575f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/682c8e6d306626c8d6df9c3cd28480295116ea02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6840d30904c03e18308ec5b00dd8f11f39367489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/69acfcf99ec173981c3a949f61e4c82e720c331c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6afb10b3ddb2255dd7351840328fd03f2c1f07c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6b33d89d0b017b73e5537deec1eabb572f012ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6c4f6f21ebfb5039c750eaaada877c213659bc26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6d97d99c0ae7b13cfb255b2ade41a82b9408a7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/6da3b59a2ad2142c166e368a389c310803909668.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/702944fa3605aceae1793ee170817be146f6d301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/71949966a49431fb77a27272156b669891969ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/76cc0aea7b72ed8769d76295c232b2d736ea1078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/7765bde9ecf4d99d38a456cbd1688af39d628d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/77be92c001dbdd62580867bd61b7a0d73358f471.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/7a508bda8762e7eb5dfaf9046b18a507fb984c0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/7d9964015c9c4f9bf97fcb2d426214b98eabf327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/7ddbefdbe5fff676f8da75bf88b90d87de7fbfb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/7f307b5221afeba252fb866f89503610072d929d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/8188a1e3f32973a6e121ebbb8b2b1b452cec5edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/83b11262e71dd6274ab6ef83df6af944e8f0a040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/83e5add317abcab9aa9a32134d40710f2524418b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/841aa71e129cb0bb1d043586208f8a8af00b8cac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/86bbede6b0955af37ddd73bbb8862079bba8128c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/8bb18f697dc2b04ccd4c13ac83779193d5f884fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/8d8211df5744234012b7fce0c12b79a68d571930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/8f96b7cb246b56a6fadcac7a6718c24feee4098a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9065556e689d88979782d4ae3a70d5c101a47790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/92eca26eaa06c410e3445940aecb867d0352409e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9538fae833481f611d32cd4b5795193cca1adbd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/95ae7a9b3a9b3c85c6740a21aaefa93a572c3bdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/96043ded666c71e4f4c982ab358fe1a6c7b9e181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9a0f74bb227715853cece36fedce151e53f4f551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9b6e30cf225f7658471ec38926cb270c8a934741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9ce190cd4e349c31781597bbc1bc958b898c60f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/9e777147fe9f557319e78d85389320f67bf1295e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a232d79cf3e855439eddaafefff977532124355b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a3ddd82c485acb6f1561ba50dbf5dcc67d90f931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a43833610982a3b460f1af90e7e24d2d6349c907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a61e165961856617a8036c3af790e2c620152345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a62f7546c0971e3d375c65b73a00514abc5a4110.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a7382207bd2c5a6d8dec4e3031bbb9187f86e8ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a7b79ef4d240f8fbc96458e3bb909dd2173f06d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/a89ad4c39ae7edfba3fb1cac8094c1830fc1a9f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/aa206e4d2b9d128536d8eb7adee5ebdebc69a03f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/aad8392008fc6f5fccf053deacaacc8009155f4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ab458a6d241c73bba03329296e19b8229f75f545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ab8d972ed23e57d56136e57f689fb22d6a252cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b1361aa6d1050bc6a08d997e702f475f6e2a8656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b20617526fc24d3814d125ac70e4cda089266cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b282628e84185f536927d6224a346313e785893f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b3e0d209f8dc4fb36b255d57fc94b0d0c9a8a717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b61545462d3a30c230731af7eeee5cc833c5bc85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/b95ed55dfb8170e83ca071eadc5cc3c4236149c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ba0951d53455c608fe932f39e307763d138f9b87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ba9dc3a1d7fcfd29397d3656832f91e3dcbf807a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bb410083cdd7c7e770db32380c16e39ec18f3d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bc6948acb9f49a0ef0b8a1582e123cb83d238c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bceb043d2a41e1fb71ba4aaf9e6021eacf13dfbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bd74e67ba97ee3a51878acdd16344049ed9cc37a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/be09ec2b5899834a48ff0ec078004af4f1c6da3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/befa4ee241dff122177e1c256b2fe79d391c1f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bf6308f77dba8c9ab8f490059e8e5398501f5374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/bf6f82042d3d29790e8b9f923c8c5dc273ab019a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/c5a2dcafb4201acf079fb5680302b542926deb70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/c5fd16dab86cc221cc90c377adce78351e2c6f4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/c77837229950ea297c165558d9b13508694ca93a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/c8caff4a14499a4ccdebb6583f49cb27f6fd1b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/cb04a396003eaf0a5938670eee97771c1fe0ee72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/cc54cc1f37005b4ce29aafa88c52b1a88a15fb28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ccb75664a859a4dc22a2cd75b732f5606c003248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/d057dc9c17fcc01f531385baf3d4ed127d4db6a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/d25226d40eb3af9360696fbeafce78df47fccb46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/d615d65d9296ef9573186491eab2b45f52f32a39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/d6ddcf866a9b1b974e2d071c1a148f3d1ac73b41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/db5d99933e22fddf42c5fcef2d2f49f65a1edf71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dc88daa82c20f7017149c3cf90662326fb25a327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dca676a0c91c2cbd388f6d23e013d546d7084f8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dd8a1f1b317d5da645bcc5c54b6c4efa6e070626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dda8cb5cb68cd7cb35b33b863283520e37faa2ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dfb99463e3f06060671f962865e1732150f326ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/dfbcc5e6fe2295b797f1e5186708e88d1318898b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e0fc58ec93c9379c3483606761500a63c9b14728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e2597d142bc5d49b419565357307a95ba146f6af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e3ee749cd3167c6f25710491a346c5e6c43bd794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e46066d06a7ab5cf6ab3cc7656aa8540a32b9700.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e4d5d110966f9edaba7e61dccf46a412f78e1754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e8e6080f3ae5a722175bc707351fde5861e0fac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e96b7a1de0fd4dc28da9276f9df14faf275119d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/e9a60ce8963303ae7b53f706884b5ad80585d2ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/eab99084612f79e804206524a4e25e291ad23d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ebc45556882a4c4b1ad9a019a3c313802a18ee1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ec53a362bf59ddbbeaa719faad872ca4e16fdffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ed1986a4f111da8670e894856c24c547eb177546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ed9fa9a1e8d62ee413a56dc220af67fe130bd964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/eff9805d8dc94594db1feda3004d2250862638b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f0d4ebdb3101ac38c08b9805af731b3e651a9b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f4414060c8101d765fe4d72c6e2ebae6903d37b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f49ef2e882aef51317dfb7947e4382691d6c545f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f541a759c13fba6e18dc2c795745cf1ff1560d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f5bf2ed9b05882023c3a343b2c2cdefeef027f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f5cb9d81fc5f670c92f4be6713ac46c5fd8c88c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f66333601e2321f4dac14224af140e5f59a68c65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f696e80968fa04aea7a68681d31b1b0e51225380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f74201b3ee902d7d580a1c26ca8edd7064404f7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f8100ee37c92a7dbd7ff1e05badefb928394ee57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/f8ca48d850a215ec2fee4b548681049640e82901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/fac47785f78300e04c8f2e9ee8ff1f9a108b5315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/fb6945d5abd5e4669b818600becfc21036af6dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/fc1477a048e4871c296f3318b7aa694a04cc30db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/fdece4eb03f87e48ff1c3a6a00d3ae776b4bbc23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ff27b6b5f90a2b95fc92005b4b94304110ff3a22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ffc83ee3cd130eabba4786a249d3c5019a0358f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/ffff74a16284a7e741e5b106dbdf0fabddf6a82a.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-18 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-18/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-18/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2015-22.yaml b/configs/cccc/mixer/CC-MAIN-2015-22.yaml new file mode 100644 index 00000000..0b99ba71 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2015-22.yaml @@ -0,0 +1,481 @@ + +streams: + - name: cccc-CC-MAIN-2015-22 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/001e68dee3c1733ef2f9ae038443a07cfdd6680f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/00464b7e0f2b8c7460fd2424384ce60fc56246e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/052b462b9cf88b07c0385024068dc65c473da81f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/05f9f8183f3980137152a52ae96c8acf0c4ad131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0733cf088b6b33e2f708da765d927c85c054c335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/08314ba4f6c3c392bca8b2489915a5d558f22ca4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0906ac9975a57d7b88920f66037ca152c900fb9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0b5112b40747f138d33f90e8fba4ced120b11a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0b80884ed7ff91adf3612b7dae6dc620681a8736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0c32a88a9b5fb52a1e644b7902964d4bf496e525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0d5decdba0031e768472ef36315e71526f4edeed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0e1fde38c6ce92892025d68e027cd8f088f454f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0f70f7d2f4eda78de87e06d9ab03a6f49001aeee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/0fcc1257767bd68382d662fc60bea2d3a178ccca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/12930032415a175c4830493e5a86653c8843aa70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924799.9/warc/4cbdae9695c5d16fcfa18086e665de7b694e0042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924799.9/warc/98de3ccc051270062df7f00bf62f408e43176de2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924799.9/warc/fae97f47f2b90fda8bc13014e636882d71cac437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924919.42/warc/e2e494ca5621bd712f0d85d6745d6312067e12ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924919.42/warc/f9e1c1047e23aa59dffc3110f5cb299e2d8d0e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924991.22/warc/0d1df86d09b864352eb3eaeafb5f7cc1ccba22b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924991.22/warc/2a1b1d3a50aa7a2e847f6f896d3fc98855934742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207924991.22/warc/5f32c4e3e988cb2d54f85872feb536d804434ec5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925201.39/warc/1fb6917a81eea050fe788b9d1eec700ed4f1a78b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925201.39/warc/93089706a0e2a95b57e29d95daac6f476b2103f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925274.34/warc/18d8044a92706d7fef27c79d524b79b0dbc2c145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925274.34/warc/3ed84ff5eaac4c0365944ee614e21ad1019c3c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925274.34/warc/c68052c2ac5af7ba4ae0e9a199f2b29069b5f81a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925696.30/warc/450e6e3b84631c3988ef7fb4ba2f51b89488ed7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925696.30/warc/8c0940b16007d00c9bca9107aca8a47ba184f508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925917.18/warc/11e1d645815715b4b4a96a81b1924056ad21d79d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925917.18/warc/c786a46db58c76d18fce17cd2e9bdaca16089737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207925917.18/warc/cd8595ef94e12460ab0e175832cbc546f440a7a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926620.50/warc/3372c531111acb6a0b1aa9e4a8081796de554632.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926620.50/warc/e1ababe31549f7fa052850ee65ac5a17aa0c8507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926736.56/warc/767ab57423c180dd2e3f16e0bea708a91b93ab8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926736.56/warc/909721d9497f7b616ba79df48093cf8e2f4c365d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926736.56/warc/da76f9be5b76edbdea60adaa37c7ca0d948e3827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926828.58/warc/1782811d01af211c90fb75be29c1cd4afac6ebc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926828.58/warc/83b049fcb51335e5b890d2b6c655ebb71c7eeac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926924.76/warc/34ead19fea0ef79f45b54a6282f7f15c0106556d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926924.76/warc/f8b834cf6d8f9a95c9985ba45cf0bc4ce1d87ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926964.7/warc/650d9a957a6c25b7eef3b7513b5fe0a9cc7fbbb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926964.7/warc/87bd46040e14fc83c16e93abfd8992e6dbb2aacf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207926964.7/warc/9b97cc0bbc91ed88d8b8c54f30a7b6221681c760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927104.48/warc/65c4e06a1c31bc3407f951f44abd40b426b042e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927104.48/warc/7c0a1be87b47d590c40b6d20cd2e82dbae74a479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927185.70/warc/390c8e11af30521637764db40279b9cb51aae55c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927185.70/warc/a4918918ef7cd1b0e89d25f3cf4acf64ec614401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927185.70/warc/d53ebcac5dc1dec3d7969bfeaa8045dfe3d4c91d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927245.60/warc/64139f7bac606db73dd0f127c6ffe4621795a825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927245.60/warc/b023bead1a109f45aec61540e8100f52652b0b7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927388.78/warc/4ccba2ae8f8ffa8a0b5714342875349758c47175.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927388.78/warc/7d221a598231e9db8e897a2d25aba68fd10670a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927388.78/warc/a474e00a9ebc87783e13de0a58fce72fdce9f005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927427.57/warc/01899a4bc8d9693d1bd076665bebb946fa643270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927427.57/warc/d37a9bf04605ac33edc4c6febfb36e4b5b4766d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927458.37/warc/534cdbdf40271172e54091d40c7fda1d459c8fcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927458.37/warc/dd7d51088c0d3352c6575f36d747fc403699c0fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927458.37/warc/e3c6a92848d823a0073a70c997c881f71fb919e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927592.52/warc/10da5370230cb1585bd8d621f557834a44a7b734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927592.52/warc/6d373e33abae0c921fa012f41ee8fd15a20542f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927634.1/warc/3cf30d08e71a0863d67ed19c3ddc5b1c602929ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927634.1/warc/6416edb8a2471b5a666668ff8da7f617469bd44a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927634.1/warc/ddc8ae19df2bba1857d48c64e32092a39f31e64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927767.46/warc/30440d27789b13026a432983b26d199751f4c172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927767.46/warc/9fa112aee808f0c49072aed63163032dc1b685f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927767.46/warc/b3275758c8d6cf572511c7a41870623037cef6c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927824.26/warc/ae3731f8d1e3848f9ce9a901f188c2acd5129e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927824.26/warc/e1abfff5abf94a7f4b2fd0e42696aff1f9cfdfdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927824.81/warc/8706e7827b69d27c6ff0e11ae28a034672abc21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927824.81/warc/8c42797f9372f2e875cda1a8a21c3d0aa98e176b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927824.81/warc/9dbda9586a6d7182083bb3b4ba618c67cdce7b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927843.59/warc/275748ddbe9a5ed189f7e163b3376f964ecb41ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927843.59/warc/33977815725abfa92d50b97dcfcd39b6674bc4d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927844.14/warc/33723b1335a7c60cef799c27b173eb6f323fc931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927844.14/warc/72631a72577631e9d85649bc145845d89af307a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927844.14/warc/f1670b2561d3c3bb06fb51cc50c8df3063a933c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927863.72/warc/18b1490bfd5385298280f6e2fd888e3af05291a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207927863.72/warc/760d2bf2f18ed38984100736aa1e773b254eaeae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928015.28/warc/22b03a2707e75091599da92ffbd8fe586dcd7db8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928015.28/warc/aa20f1359640b208d5fa18d67d644958835e8da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928015.28/warc/fa4832c0d956b415be854dc7c15dd38ec3093245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928019.31/warc/28c2044c236ea9d0749dd0c4e6955201fc84eb7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928019.31/warc/f1a6e103b4bb784236eab8352d8199924679c7e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928019.82/warc/51506c7410147bf53d5fb1d52330b5dce0c169f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928019.82/warc/67df325c1b90d22ff6dc932c516c1a5557e23aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928019.82/warc/e7233cbf11477a5d50892abc005b73b1d5415c45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928030.83/warc/78d36801149e64f5b492a8f70c4faa7155e4de3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928030.83/warc/7f0cd74697f7bda76a91775a914000389527d0a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928076.40/warc/15810c1f723f068c797aed60813121dd8f1fe4ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928076.40/warc/f657fed2d5319f0f747c91b4eef6668825f38b0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928078.25/warc/086017aa4d7d62f3c85750671d21019de713afe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928078.25/warc/0ba37bd50a50e439b410751893e9114d9fc8ea06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928078.25/warc/d10a984d87d398d5e1020c1cc01ce67427bdf46c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928102.74/warc/402da48ae21596395b306cf42b9cd5c8f2b1d5a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928102.74/warc/79333d677287b8d2b877c0567037d40cb376ef04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928114.23/warc/5c1bc1b6c7fdd74282c0b0db874e0f670bfa0d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928114.23/warc/c6258318256d89233e1c04d144ee51a9531eea35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928114.23/warc/c68325ddeeca6a8981ac9b031ed1bf1e7249176e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928350.51/warc/282e547ff99e3a614b93a6a37d569965e35c5198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928350.51/warc/7183a4fb8a504d80a74a3b77ae865768ef88e3e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928414.45/warc/35946f3d43409b78de778a631bb1875a0b0e87b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928414.45/warc/521d5e7f6ab2d44ebdb6d3a45e4328e40dfb59f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928414.45/warc/d4c95c1dbb14f15ae6fa618759a8f3543020b99a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928423.12/warc/36848a0067f7dd2615e27570050dd2add38978ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928423.12/warc/38d0587471c597ca7057d160c880dae7dfd51f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928479.19/warc/225b0b19e9887ea38911ca9d58d142daa05ecee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928479.19/warc/50333b11b7e3fa8ad96ded7d9ada6744f9859748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928479.19/warc/e8e6fc656e84a5ba79bb5fe18732c194e5820fe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928486.86/warc/58582f62c0492a526e3f2e4a5df4fb7944f81589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928486.86/warc/9e7b87d1dace8cdd132e7db9df06aa1772022c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928501.75/warc/2b35f5bcb7f545a7a755306a024bbe30849501ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928501.75/warc/aaf1f890df269932e42dc44df10466172fd03a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928501.75/warc/f15708f7ebbea4278731ae650d61761a6b11e829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928520.68/warc/8ef810cabd5e7ae3a61a4b577414b4b58c5d1e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928520.68/warc/dbf497adf6b71d8cab333801c776b8e50b9d9c9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928520.68/warc/dcaf756ce0bcaa129fe177bfc7373db05d1d7b24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928562.33/warc/a7f2a4e3ac15af9256f895fc165c7ee65b243588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928562.33/warc/b95f8eb4a1bfce63e8afbc89f74c3d0fb9d34467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928586.49/warc/0740877f45c5d2628fdf08452d790226bb21e708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928586.49/warc/a47f3f69c39d50919c82f28d4a7ed6ac7eb96c28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928586.49/warc/bc9074303a3bb054abb89885e6e2606649ac5493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928715.47/warc/164c171890f973989f959125628b49c6c4344ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928715.47/warc/9faf197b9bc78b4d7c714b368a40f42150e3c125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928729.99/warc/344bcbb7c7935452960a6078c613208d0993693d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928729.99/warc/c0b9e823b6a2514933844866b5c3dbf7fc1e0254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928729.99/warc/dc90fe76a04667be85039d4d15971d70cbc67c82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928754.15/warc/9ede69b996acec9adc7e20a9d924615a8f51c5b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928754.15/warc/d56e06d235788e3e1d55ac66613657acc1408127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928757.11/warc/29e51f35dbd8fa0de0444763fe4538a14928ca44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928757.11/warc/8ef78514facddbbdbb8221a533875ec64fbe27c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928757.11/warc/c52918c8284c1caf72f2ad0206d0cb5eddac9380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928780.77/warc/4818b2302f7f63fd7d0d15b240e80af8fd6cbd7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928780.77/warc/8557c14590f9014504bc37071b15992c4f24cbf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928817.29/warc/26a11f77e895cace31df9fe8dce8b58de1d2d2ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928817.29/warc/2ea64f3e239bac48666f4b270f40955c17e64bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928817.29/warc/8866a8ea58910a2f8f5c17dbabee09cc5f78480e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928817.87/warc/62b5ee68cd1f89b3a7c7657ab1ef8b55e4679ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928817.87/warc/b525618722415d989f7c4ed58d22e6713961cce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928831.69/warc/4d04c86a7ad429c7916f607504c85427e20f1dc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928831.69/warc/651232eaf6002aaaeecc5aba19155594fce7c596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928864.16/warc/b288bf0abb329bb3cdbb322e056e6b50470c9fab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928864.16/warc/c7c3fbba0afd99f7acdeb48f32935e4d0c4c2950.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928864.16/warc/e510eb18634af04bbd1c167073aa6cfecf45a693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928864.73/warc/0693611f1c02cd6960a3566036cbf995155ee6ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928864.73/warc/4216db4c4570a85df1e992f57f983a59ecc32ecf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928865.24/warc/2f27bc5b1aa5292e4f041d37b164e0751f814691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928865.24/warc/6fd97804b4e4862314dad6dc9e552491f7a10fa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928865.24/warc/b39d7c054fc20d09a8e2b9b80b6734345d2cc717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928869.6/warc/b4c5c566511b4d1fe3c8214bd678c250f85209ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928869.6/warc/cf91ee2edf1dc526d7ede1de6f09e08f6453cd33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928907.65/warc/18ee159f8b9f00cfb8e4158a486002875338f46d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928907.65/warc/77ea1719e91e6a61abade482e20e788af513cd6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928907.65/warc/7e012a1cf7d1eb45337fa59a84f54d751d2e8f65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928923.21/warc/0b5fe1d6df5b0555c46506ac69001a2f3e2bae72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928923.21/warc/3ed8b0bcd83d193f53788a339087f475c9bcb565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928923.85/warc/a1a8656b81339c7771bd1e3e0fb8a25e0d790283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928923.85/warc/a7d5aa3cc84ca7312b988971e20b6db6f1680db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928923.85/warc/e1e7fc230f74cb4b18f71d079377e0a18cd98d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928965.67/warc/9ed6494fe4276b5a9100028df9e8945aedb3cd95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207928965.67/warc/bf7fb39706f0813fb685d3aacd643789a113e7ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929003.41/warc/1f6f1e7a2bd4389b09d1195030e8c15444765c7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929003.41/warc/b899d2e7c8bacde5328fd78b2821caebe293785a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929003.41/warc/df8980aee00473dcca4856fa09dee4b70a1662a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929012.53/warc/3ad666e36ea002127f893d0d320ba63d84ea5df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929012.53/warc/6b4e259958b548d0ad14828cd573773baca73ec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929012.53/warc/a5efbf4b80d15f1cc6f878fb352c1109f562d0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929023.5/warc/4d981ac3c9a8b2e15babad5282df752ed4b6f57b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929023.5/warc/69729de8fbd2c17811bd86e2ba6428c03c307820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929096.44/warc/5c3a3e6a38f2095b6b708c8dd64014fcfc790ed2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929096.44/warc/6a81e4978baae059b5955cc840f784a27c4b792c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929096.44/warc/eb2c3d3baf68b184ada01591e2caa9b0efe429c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929171.55/warc/0d9b7b4c30ed5797ba65ec176d3186e1f3dcab0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929171.55/warc/1d630dbce69d985138f97da241f565a2f7ef0b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929176.79/warc/327f0bd23150bfcc97c70f00ebc1087129f0e0cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929176.79/warc/4e481a35fc83e560250cfab272018f8e75512156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929176.79/warc/cf5938b29d809dc7eafda1cac21275bf06f54e8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929205.63/warc/42838845481b8bc93d56547ca3ddb0942a6f5167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929205.63/warc/74460dc5de2916bb99e221b0c1b3d3b897bcdfe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929230.43/warc/4f776ea2325305928e0e85395a456acb5a6a2c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929230.43/warc/bf53c61db7e17ec96574a9d264883ecb29bfba42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929230.43/warc/ec43d42c8ee843044e1c05a5ed3a91cc3c33143f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929256.27/warc/9d0fdabda41224313a67f30eaea6cb85b83b8b66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929256.27/warc/d97e0579d0fcb02dc4ed7b0f59d698137c579f66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929272.10/warc/2e026084bc70054f3d6e4911d7542fea13262ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929272.10/warc/5c016b82ab3fa015b4f7da298c5544bfe602c79c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929272.10/warc/f48d06c69a1038238501fca8dec022bad4f5b1b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929272.95/warc/1bf08e82930c925309835ba74b8563ccd6d6fa77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929272.95/warc/8f41e438c404bcd2c00ec2c904ee99239e3a7c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929411.0/warc/7b9bb2dc85db09cbe8ca87b9520c81d079e0c088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929411.0/warc/a7ea1cadf228758483e4e5fd3d824f62b6a769e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929418.92/warc/ab06a32b46e80190a373df0f573b612abb2ed02a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929418.92/warc/b4558a99e4e9a2f269d6d3f5dce4821aee127953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929418.92/warc/b518e124d5747f7812b154864f1e7e4fe926f856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929422.8/warc/649757b628b6b952fd029ab004760feab2e05a7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929422.8/warc/66785ce491b0fad9e72ae7a733a2d6052ceee409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929561.80/warc/54d949a5fb3dbdc077d0231b5460d84df7c76684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929561.80/warc/917b689110c0f3aa931df8469324e13e46afb796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929561.80/warc/a574a71246904df70ef403dc50819ac4942d9d4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929561.98/warc/e26359a30e91c68b92f772c094565bddc6069a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929561.98/warc/ecc8dc3070d203c4dcb301fa4a57dcf591db435c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929656.4/warc/22a48e3ba6590c21227f21df53d0273cfdd6f510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929656.4/warc/5c6fc4e7b1af7c37e01483b70985ed739ccef78c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929656.4/warc/688ba20cd57b55f63999670e2c9addddf28ab503.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929803.61/warc/9b604313bde684b913cec6f5ab57a5f533566d80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929803.61/warc/9d931ecdcf78e44ce5dc56939d7520ce28cbba6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929832.32/warc/0d5b6b55692d5f8cd0170e5000fe1a9e7df820d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929832.32/warc/2ba01981a665d4126edda8af9554396be8c72beb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929832.32/warc/c99648f11d42f60988356badb5ff222bd02cf6a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929869.17/warc/5bd35b2b6a7fb6ed2abafea68cf0a52f1e7ce36f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929869.17/warc/a0e88d9a09e4f531ad8e77df631ad8ed2bd1c142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929899.62/warc/5df85c6182a89df695122170bc64201b2c396aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929899.62/warc/6877751c3b76b6c242525245a2d8ac7874cc5047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929899.62/warc/f6275228ae8f969a4a3b88f130ed4196e64b2a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929956.54/warc/1090955e010986d45b85d4d252e969c8c842c3f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929956.54/warc/39bba298476e414dced809683d6893fc5c0d9186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929956.54/warc/b384484600d9a9557c991ce7ae9cb0249c1bd819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929978.35/warc/20eeca009bc32e26ba1f0d8b5de48b24ebef27bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207929978.35/warc/ebd1a883012398600c606b57751177d7c5219751.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930109.71/warc/10b331d4d28465f5c230192a7d9bebdabfeae1a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930109.71/warc/121aaaf4481d3239a27182c604a24b842124d18a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930109.71/warc/71d99b9799919d238a8b615ece418ea7de299b4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930143.90/warc/3750a9b377aa6afeae6ba553ca8eceb41492675a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930143.90/warc/4b66d90e8b4732f3df0b61b2fc630aeb6ce0ffad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930256.3/warc/169e8f8142da48676eea5f86b60e02b8aecd5f3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930256.3/warc/1c37aafdf16322f2372bc590d1165760174923d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930256.3/warc/9e17d76f000a2023d4609609ca18c575c8837ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930259.97/warc/81eb6ec33bea4042e34e6628a37c3a6cc958a21b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930259.97/warc/92e901cebace73e443249fe3b30887fba6bb48b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930423.94/warc/3bf34996e6d948d158fc3837c15c82c31dbdc167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930423.94/warc/925638e664149e8ad5d4ecd8f608d234451db232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930423.94/warc/e2457d177323d492e9ed0b4b5cc56d982953d99a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930443.64/warc/798e9b914c5e53d87ab659b27c8425d02c331374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930443.64/warc/7a12ff147b91f70c80d0c362552e6528aece95f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930866.20/warc/2086a222468f82133600311998a17d2119c2e398.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930866.20/warc/41d70a0c0d3288045af7c4b28df3c3e31827312a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930866.20/warc/6823a8a49b25e8c2eb5d63d87b661508a53cee3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930866.66/warc/7b011e441baba6bb51b43c09dcd91dbb5197aa8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930866.66/warc/ae756e2f4f926e1fd0dc286f2910962e6a47f244.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930895.88/warc/21e6581896261a8779fa3d8d9c328917e60f283f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930895.88/warc/e762e0099582fd9ed8a0378435c67e9761079c7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930895.96/warc/563a5d3b774fb6e41652cf405445ddfa886877fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930895.96/warc/9c85d77764ff6cca04dfab29d233b4b29c387251.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930895.96/warc/f7d6370d250ad8a12395339f57f779330b3431a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930916.2/warc/2af066eafd175ef14c57d7f18c92b13d318814b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930916.2/warc/525d24553d77a8ca02f166646b7c8ee8366e12c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930995.36/warc/0c8a02969c0d7f94303096f3e576df386f949689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930995.36/warc/320cd2d5dee698297a549b2f9e171cd5c6917037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207930995.36/warc/61178a90a70c814e0d03008b6059e856392d8532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207931085.38/warc/2c0677348e751a5d47c2d85129db9b38e55973a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207931085.38/warc/d3b6edd62cf4dfcf7314755c4044811873145de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932182.13/warc/3e820782c47896d92b64f24d9aabbd9fe088837e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932182.13/warc/c895b37957f2b3d6a9ccf4496c13b1c7c62203a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932182.13/warc/eeb4ecf07105a017a0a2d560e3842cff86bc321c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932182.89/warc/d7f2d928d06e6b0f63e63604259649a2b69a2309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932182.89/warc/ed877de5b34dfe1c8cbc4c86381e455f585bdf32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932596.84/warc/0a8a7e698ce180732e4be81840aeb0b783fe3c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932596.84/warc/3a33200d52c96c651d3841edfc426db38de45174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932596.84/warc/b0a68bd1e9afd1faeac6fc014f7846972a09ac94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932705.91/warc/e95fb72f9e8b6de95ca48c249d7c9c3d8263d758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932705.91/warc/f5fc36394a37dbfbba5f05d6b575f4323d6b3f5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932737.93/warc/a883d05bb79311d6b0eeb4b01fe0d2703980bf9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932737.93/warc/afcfeab5db21069b5183df6d6a3d82f0742a674e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1432207932737.93/warc/c7daa7814fcb5e9d15aab0c40a36ab986f7d141c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1433195034286.17/warc/de8773703035c4d646caf45a9adc85169cedd032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1433195036112.6/warc/ff65ecd1a01c1147323d518a57f9eb944e2164e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1433195036702.22/warc/f755adccc0bff1556665f2dba59a57a9ae02ed31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1433195037030.16/warc/faa4903d46dc2623cb57080fcbe9e42bc21866a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1683a84d07647090d26a41a19669b9c1c6a849bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/1996992e75e2d67b58e7b836be991cbf34c50f71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/22fa57642331b427bf641cd625e0316435a0dc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/24b0c926b749a0c6bf9ea5ac01f0545cbbdcdffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/26173b572dab7d6f39fd7374e13e2a266b118fd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/2659986af84aa88fb5d1d3b2e5213d35f095373f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/28595089cc12490dac3b8915348fc790d8e57c19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/2a833bfb76c216948111c2c75840b09076361517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/2e14ffa1d03ea34d0e942faf7119e22634aab244.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/30907782e23eba620794babe47375282bfe6e065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/3340aa305143216ee92da9d730a228578ab0618f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/350621942ac6b3b013a8484cf5beda31ea4e4993.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/35ae720b7dc032df560b7dc2cfc198724b06f57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/37c31126cba90579037486c89be35e8a640689c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/37dcc2d68df2b1b97b32b348ab88fe8e07c9237d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/3a6a60c513e8f9adc2bc177d207190f1acfcc2ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/3d653011d0dde4ea6915adc36ee931f5063205e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/404d7f439afe3ee7fe478ee9499a8152889b2699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/462525aff50182e9241a13832c92607edf4da1a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/463f6ff68d5c8a340ae616b90a4707aa194bc75e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/466e14b075751861b5984cbd76a11cf8fb7f1307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/47812d6a3eb2c968fb5a5fa9213c0ab500b25c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/4a86f5f8d5a02abc34aa4b4ea71d742e7e188fd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/527270919fd86a4ddce2f1478f6b02eff731e531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/55302c30c97b9fce5805a8090aa004f0637771a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/5f840106edb11d428c3f7e703b3ddd74313b3452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6438bce1ea54f6d002780486e0bfd5d3df985bfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/66777878b87c2302084b17b62dde5973ec186da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6695eeb99c97d942ccfbce53f82f10c8830bf281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/696ff14eb0366d22181af340f70469afe2ef4c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/69d0e5c4820047692b684b9a736c0c0b1b125ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/69ff27c8b407b05af35acbf20c139c6d9004d7d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6aa353daf78967ece7507fc3b1d69f0a7f26d2b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6b9fbf0f87e50df6d6e538345266c49430779e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6c027f2ed2ad73b5959351fb66af13b78e386a9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6c3b7061e82db7862cc74f4f45cd25e02d012f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6cde3a23b58d4a72eb714c30d5ae7b6a2acf68f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/6f99d18c04a8f3edfe1488ba74b2cbdc76e079a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/70a004ac1a8ed0cba79d949d72f3f6a0f48fb7ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/70ff50ce2c1ac093edf3df49f495d748ef12ec45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/7545dfce38a53be126809f06c644bb4fb91a19c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/75d424f7fd8c2742585f108be990651ab30ff02b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/7a7e9448e6e8fb51b371d4502177286ac1a00f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/7c23f8defce90d037c3bb916ff2f38631103996a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/7fca4c63cb0bb28aed7d16b86d96550bba8619f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/7ffb606e976f4fa11674b7a91cda4a3f4a41b553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/8268e8d8a3d418e38fd3a6863cdff6d2c4a5665e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/847b37fb58ae44acad204f6ddb26af0d10b435ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/8b8a7cb74969b82b64a02c034e32cf7cc835ae1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/8e0484dcdee9750fa957e9c050787c4fb12338bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/8f1f8664a8ae82dcd8ba0db5fd0ac94c7111df2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/933aea1c92ba0ab70bf801a05c0f90007e26de27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/93ec6d3314f46a702390ba3598bf61864a06f2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/949c4ce689bb701532e1b915a935b24b1e6650b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/950859a60063b7e62b7985aae7af5b8e689b8bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/977c0475f5793410bdd3b5d38e809979a54ea8cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/9d5e0644fb3976b61251d9f0e2f8052e9bb601da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/9e7856562f62a46ab5605ee48c2cbe1a56820719.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a06a860b78de409f9eba532af07504c903af557b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a1904a48b82a3a3d469c4384ca9c12b83705cb37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a40a5090d1f708c52db0c1427c1a7c057e29727d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a41eeef83a67f1411ed5faae6a065250d29e102f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a7670ced250d105e292b19dde48faddca67d8527.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a7e73e85325b4272b2624c5f5b45c8b39f2977d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a7f7a3126d0c503c5df08d600eadeae0131f2474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a9a14ee3bee9cd7e3a0e8f3aba787d7ece5b30e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/a9fd00a0d55fa6a093857ae808763d75416fd143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/aa3e50b7f562e4f35be0115cc8aaf96ca566fcf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/ae3f3e23b2dcb135faa32e2eed7e4ea85b667499.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/b1b8e3731ed52b0350a1405e0f02047b5bd11e6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/b60f3cc03015c1faee1014c568d9a19108664caf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/b7e7aefa0a422c14f9210b5fdf9f479f6e3dd667.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/b9bbf61b04d60aecdf73df02d0e41e01d3914436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/c1b004686336d614a635608d2d36c7ec4889e304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/c4b338e9a96bc7676c14c86fef274d2e557692e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/ca0dcc53c9034a1d585681119400279b07d6b64d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/cac6a525749b783638a96755e3f57e8d58a5b935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/ce8befe68f21ac9711678ce98df8d1ddd153647b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/cf01ea6bd5adade4afd0c5e7d6b2a2b2cd06095c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d00e6e4538e2a6aff10bf53dfaadb7c6451948d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d14c03ce48c323d5b6d842de645d466b72fc29c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d477f09798cad2a2070f61e20034d7ead56c6e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d49140f567c75e60e55cebcdfde27367ab7e7d0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d715364b56a05f93f03215cd7ff300ba4a39a91c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/d73d0223d8377c6647c2c91d6af63ac2ac1ea9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/de62338d9963ff6910a2b66db425cd014f501a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/de707310659b56c3915c2a09eb10503bda7b91ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/e45708721206ec2780e7d4c0b369ca1b30f8b75e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/e463c51b5163446753d97145c166c1be33a130d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/e7f0dd50ebbdbb4ae077afe6960442d9434db977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/e8ee6dbae141b181206bd2c8fd32cfb002b49ea6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/eac1ff08657f1a17607b14531af6efa6fd6da977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/eacd69c052d0420fa2151e23e0080f9e33b80cd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/ee1daac8f77b831275a6b8f09abfc921b5c33ba1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/effeaafde72cfa1c86f39f9a098b6868de003adc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f034bac21987537f7de3b150b7bc9f082fdc7abd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f28e3d2a57c318f36f6c663b361bea7fc51e8793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f337ca711f8659688b473cac487794364dc51e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f5d9a3356dd92b06dbd62121b3e2f6552376f8b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f76ec1c72a5a9aa87ac96bc1ffb4fc2711dfb415.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/f997ab1ab76eb5364452a55334e244ee7d1130bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/fa49b8294fefb282c0ceb5587a5bcefbb9693044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/fff85ba2f0f3e94d384227bed7873708347c5509.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-22 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-22/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-22/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2015-27.yaml b/configs/cccc/mixer/CC-MAIN-2015-27.yaml new file mode 100644 index 00000000..a8d2749e --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2015-27.yaml @@ -0,0 +1,419 @@ + +streams: + - name: cccc-CC-MAIN-2015-27 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/00a8b03375316a4fbbf4f1b9c7b0ef32a2dd0366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/08ef7fa74c0df837fd7db18814c7a2c5efd2177e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/0ba99886d3c1021fa8cb7a393143fbc9937b5374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/0d493ee1d154dafb683f82c13b0baa7e0c4ee149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/12ecc5f2a72d9f0f29b1fd7206d35db56e18e161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375090820.0/warc/2f1be7414feaf0eddc1973ca2f3a47b9948cafb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375090820.0/warc/5c601a9935fce92105e6003ec4d2e4a6b54106b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375090820.0/warc/cafcdba8f216e5c1993e5cde1bc2a599fc0aa1f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375090887.26/warc/0edb919d0c4e7f00b3fa196d29e864158d28a435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375090887.26/warc/3e6d35991a6d68f5cd83d3f447a06a06a9e8a9d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091585.27/warc/8dd3a25e84a17aec1cac94460baf3d0d84d0368a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091585.27/warc/a6df9e22c15a9826eb8572c7048683d12137aa61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091587.3/warc/8208e81835f2570ad40090f03f651d50cffa837f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091587.3/warc/e3482e36f2df6f591484beb81a1fd9df1a98aa03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091751.85/warc/8e84159054866146b424dec7a93d1bd747f9e5bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091751.85/warc/c40c7f4cbfd4700215b78dc81302af26d6e12fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091925.14/warc/1aec0cc8a0c90e1f33bc2d6f9214a731b4b6bdf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375091925.14/warc/838a994ad1c6f3503ddd6e7d092dbcc9e430b9de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093400.45/warc/c65d4a77e7cc25bbed4b8f2ae977aa5f5bd98801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093400.45/warc/e1c3dc62c1b9416aa9bfd848a591394f6c67310e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093899.18/warc/4d00c772e2becf134b7a7201700f0472b70274ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093899.18/warc/fedb848d17f1002edd6e000cc7b66259c1f35601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093974.67/warc/07696fe96ca5a7149502f8379b2160f1073a3359.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375093974.67/warc/084fbd7d258cb77921d282b6d2ed7694d9fd41c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094451.94/warc/1e2ebac6aa4c40a20f7754036c6840a8ae4157d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094451.94/warc/6eb7ab13c5707e7de710c45b2d9d1f5a03e75cdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094451.94/warc/e4bd5078b8f12cb765698bd7eeb5759339c9a69d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094491.62/warc/a775618a8c7877772010cdd95a8c12553a8560c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094491.62/warc/cd4308a92c45d64a5c6763b5e2f3bde268054ad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094491.62/warc/e57bc7ddbbe2b0decccc92ac8ccd4d23263c22b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094501.77/warc/63b3de8f19d006d310afe1b33525878a436ade64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094501.77/warc/e22af352c8d403f4c7a3de1e1562a4d45488ef1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094510.84/warc/73e4b5152cca7ce49b016d1a2f90fb13e670168a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094510.84/warc/8fba039e9f968ed41033353565bc7d9336f7f6a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094629.80/warc/26f6a5e97a9e129410a708dfedb3bbd4d4e71635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094629.80/warc/3d7ebd6d413e8b550adb6f00415fadc4d34c4f37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094634.87/warc/19aa5512ef5d939fe277f2412f81dbd38e80bc9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094634.87/warc/bf08ca7547c9c2f60fa7fdf76878e1885d001789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094662.41/warc/625ff8563f0eadf04fdc7c05d37a400ed69d5604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094662.41/warc/9b8bcb110c1b0a7bdbd08fb4c070c36efee0c8f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094690.4/warc/6c2bf4ef482d40362aca60df5b48443b9b07cbd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094690.4/warc/d6ca0f595b69aeab40f28b92acacd0d26e448776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094735.50/warc/f918868c027864ec4e8821ef998c6f5d96f85ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094735.50/warc/fca3a2062b54615f6d5e58adec9264f664f83107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094924.48/warc/5764155757337779aad0554cf7930a6a0d5544b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094924.48/warc/8299bbdd6e9d4d12cc2a118bb807629348d4efd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094931.19/warc/1903d361b978e23b3fa3a241d04c8e5ba614d02e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094931.19/warc/2ae83d05bc01ed18a4ebc03747af80def66e0f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094931.19/warc/ac3c88e888422de9a81370a68336f64baae45457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094957.74/warc/c74502b422cd8220807ca22afd3301fe9c228b69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094957.74/warc/ee0d478307c20e06cc69076721cff8b169cab863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375094957.74/warc/f20b78a32844f5318a7d8cc89604517903db95ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095183.13/warc/5ce70835cde49a505f1b820cb4036bf23f4cceb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095183.13/warc/c99830168e0f355c769c51a6b1b7237b334ca453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095184.64/warc/afb639d81fe23295de84f5a26ac472f9f5d99d98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095184.64/warc/c792da577830e0f22121d7477cd156cd71f95e6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095270.70/warc/5f1efe2be292ebcce009fb92956b6ddaa9943d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095270.70/warc/84a9f4c79f1f42e58e2fe6791e9d5101f2c9503f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095273.5/warc/40e50fd11ef7353ecbe8655db6bc75dbe92fbc52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095273.5/warc/c98dc8a3763dda3775df813c6458d76479cb9422.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095346.56/warc/59706ed603f70b83e2570d622ac21af0187eb724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095346.56/warc/e88565684b23d08e9d08791f5b8f82daf126e37f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095373.99/warc/b144f36daecdd2cdcb00cf35f0702831743c2e5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095373.99/warc/f15f954488bdbeb9a7cc7552bab2992a6562d885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095404.78/warc/03ffe8fbb3e50de18cf1fd8616d189e947a284f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095404.78/warc/de7ecf826980a2bbff8f750b697402dc85a6ea07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095423.29/warc/2bc3189475b3fb061c0feca3a52b0e5113c038a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095423.29/warc/2eb57f45c08caf30fa874e71775aef919687c7c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095494.6/warc/1ce6f8d853f0b4218722fbdc54734b860d7e10b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095494.6/warc/30be5ec0b60ff834ce2f54b0136850b4e5f21419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095494.6/warc/f394a79478156f5580aace1207fc7d47f89aeebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095557.73/warc/a95f674c20225b42fdeda1c59fa7402a37196ea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095557.73/warc/b33921f5ee3dafcf6e3eed14b6578725ed60d230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095557.73/warc/b355f7d070c819e98b8c8091329902099096962f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095605.30/warc/1e97783d2c835097137c7a7fac447c9968aaf424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095605.30/warc/ffd26baabf9d129dfb149b93088b8fa85037b8c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095632.21/warc/33356904958d606eef90d17a7294a0b8d91996df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095632.21/warc/c05c85231d2f723cdee8e3aead17ff833603d51c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095668.34/warc/057be0614265884de9a218008ffac4cb45fab45b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095668.34/warc/8c8bf2e8098e57acacfc4d165966dd2ffc7401c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095671.53/warc/27d6061fc1306215114f877f0ebf54f8a80ce72b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095671.53/warc/efeebedaafc5b4400be7e31e741805ba8e405647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095677.90/warc/afee2566d1ae190b570b61add8128350fa6c9cd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095677.90/warc/feaaf29d8d00abdb2dbd5d245b36bed62cedafc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095711.51/warc/7feced0c3f07c09ce1af4b7b579a108c891a962d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095711.51/warc/f0269183c12b3493dbe0babdababeb11f5406403.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095732.23/warc/616dc894808f4131ddc884ca1005b0a846240b81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095732.23/warc/fb61c1f1ddb815368feea4a200c7b49b554bdd81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095775.68/warc/dbdcab78634772b2d11161b75dd50be957422a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095775.68/warc/fdc29f6c475848980d948d6864ce5ce90d5aafce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095806.49/warc/5f0f4caf9585dd437d879999902f864d5ec8f922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095806.49/warc/86f096ca40253736b7db244d915b9d3cf881debc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095806.49/warc/fead607f71077b30b03d5e95be6b7609076579ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095874.61/warc/0a9741a796398711deeb5a04a5edab254ca7f8e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095874.61/warc/40e6ce0b24426b52d53c074e1e1eaa90f86d92d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375095874.61/warc/7023e25f45dceb8c8abfd0c4ae74dc0c040a7f33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096061.71/warc/5f52cc075bf77e7abd5a0a63112a230ffdd2fcab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096061.71/warc/7aab8c50304be054504961a91623e75e2d6e0b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096156.35/warc/9b912d86fd79b728e2d1fee2f10e82c9c989b902.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096156.35/warc/c379df4505103d41763f35e40b28236ec8b4998d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096208.17/warc/10706ce2f6da2c321674c38f6e98d61865403dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096208.17/warc/ccb3cec1deb8c994f1af5e1b33819643b4243863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096209.79/warc/3aa9924b5388ed8c11f02e42f558b59bee14f27b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096209.79/warc/5c72852b318da244b63446626cd6ae18f21b824a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096287.97/warc/475b0babf911572ff249fc42cf9b7605292489f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096287.97/warc/edd0fea0ad471695d49e6f10712e3b7b39b26b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096290.39/warc/1b1807062e511a1e4b71db9e0e539f7a81eef082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096290.39/warc/c935adf7ad0d0fd9d3d240990808a20cac3af786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096293.72/warc/7367df5762a878b4bed41caf4c19f0cb00220eb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096293.72/warc/7abdb88e468f67b1fc0f79ef6eff308b22e19a9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096301.47/warc/3c723625ac393649c08785f32013975dcb35cbc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096301.47/warc/e70e3bc9689cb2305bdf48025a23aa403b8368c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096504.12/warc/25d108f0a11274fc843d8abd38df6bec83194e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096504.12/warc/39c23a653bcc70e4a4c296d65aeeaf506c80cc15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096504.12/warc/ed0094fb4638c5925ccd411d7bbb292200d919ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096579.52/warc/07311632682df0db39be1a6978691fbd81e1200e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096579.52/warc/a6839d3288901b99b558846bf849ac95cfbf0149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096579.52/warc/e9f091c9dc1c7a0fe85e946dc043014b38cd4a10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096579.89/warc/6db7cf7c1ceecb54ce1704e9ed17919bd98f0e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096579.89/warc/97074539898a70996fa5cfdf58d2c3cebe797edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096686.2/warc/082bbd2bfff03477cf337f66713f862ffcc8a3cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096686.2/warc/c47ef3738c2beee25182aa4384973b3d410d77c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096706.9/warc/3e16267a068ea39d20d8ddf8ac323b6ee10a87ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096706.9/warc/5fa5427af850c5476aca51e4571745693455a8e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096738.25/warc/4fda5f1e78190a6b0182632b2fd548582f16c6ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096738.25/warc/e3d84f3ed5eca5e874be760b73a5bd76c9a60668.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096773.65/warc/42cc5625492842b0a07c653eebb571ffb3fdf766.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096773.65/warc/510dd346ede0dac0c2f92a289b294b8d7b86859a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096780.24/warc/21621d96a409f590e35320c1aebf336ce0d968ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096780.24/warc/ac4749d2ee7a0a7e3eaee839af45de20010c458d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096870.66/warc/97db6f03e4089b5ae7445c39667f064eecc4bfda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096870.66/warc/9d15a2babf46e0c78500b5a6e6da26abd4e9c3e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096944.75/warc/6569bc32b51910112304664ca5560922e3b4d13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096944.75/warc/cdf8bfb25111a30b949cdd58b493908f352286b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096991.38/warc/2b0a49007a8b0b066cad61ad1238f2294458ce38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096991.38/warc/9fa4df7df30a5cbe5aef8a57cfa5a5dd7c6ed520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375096991.38/warc/c7ac8a0cda463a31a3fa4d3251c4b2b79b20399d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097038.44/warc/358337053388c03679f68412ba6971d4c17d025c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097038.44/warc/5748440bfa1ecd38552eeb910c441a022bb11e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097038.44/warc/80ebb80bd71dee47a3836281086121dab64e01e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097199.58/warc/011583bb1ac02d4a8b3707a54cabaf8c15f09cdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097199.58/warc/ac076d1dfba9f4551e135f433db4182b2d5d4b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097204.8/warc/b6c1e390bf8d98b4423ebdfee4be2aaf708d1923.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097204.8/warc/eb2929b0dd34f7364d54c77c1622c5fa6e87aeab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097246.96/warc/3e13d11f772cb9895cea1f9fbe6f628da177b523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097246.96/warc/50c5bdf1f78cde653db1f1447aae7e6790d5bbed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097354.86/warc/77cad681c44136b2f7d88dd405b17392d2a9b3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097354.86/warc/c204ef10d2a50deb81bd98063263b89f4135d153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097396.10/warc/0fa65598a255103d49982dc02b2938a91153e6c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097396.10/warc/f775785fc0be1911ff703e285693044ec86f9b59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097473.95/warc/313b0791e1263cf8f5e418ca0a29905a730a3b9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097473.95/warc/9bf84ef58b5c5d3c445ec2882ab3e955b97357d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097475.46/warc/241800568cd87363fe802b1ceeee68034725966c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097475.46/warc/5f2d38e0ca27a1d102f415d5366419815a967dfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097512.42/warc/14dff3885d84f0e16b6bbc2b7eeefc24718b26ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097512.42/warc/2f604569bdfa6f705cd47e84ac2b779096a6f39c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097546.28/warc/2d071d44f21b0addfd120c3b592a76f3bd76dbc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097546.28/warc/854569f0bf063043a171429e11ffca6cb5d24d8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097546.28/warc/bda04f23be0431783300d7161e4165787d06b062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097548.7/warc/1cc9aad31e7a8f127e3543b53581db8102e327f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097548.7/warc/3cac9a945058be583a451903eb292243df5ca530.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097548.7/warc/f3cdea56a81c1b2d6ad0debf31cae1462c99ecc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097710.32/warc/01d1854cbcfd1f1981134369621e61abdc305178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097710.32/warc/cee9a3dae2a32f775cbc0f89f760ae8b78849289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097710.59/warc/01f3294088725964bf31866a8598c48703e1b227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097710.59/warc/8e319fbfe3a55b5684f059d73aec29fca70b9230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097730.92/warc/00d515b1a406a8e9a77cf24b2465961affd0aa4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097730.92/warc/4825a30d352b677dc24e64e65554e876b8874abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097757.36/warc/1836f8a7e320719acf302cfb9a1271dbe11a8cda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097757.36/warc/c3d12b4b88c3e7a66389ce4379a9442d6e42f89b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097861.54/warc/70aab1e2a6cdf49bda9b7114e300a3ba16241783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375097861.54/warc/87269530a6dd3e9e444df9cee665b3a3c0b5420b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098059.60/warc/cbe795d74b09846011a28eb073d51ef64de90bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098059.60/warc/fea238db2644c393520c0bd8e2d8c26f6545ea20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098071.98/warc/bb18bd0421434dd65b8a78abf486320cc339f440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098071.98/warc/ff9d2c70f83ddd4157dd1d134261c18d9de76112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098072.11/warc/93102066cea6ec8094657058c9547a1199584b43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098072.11/warc/b96833fd9244f57b2843161b5f608b1375aa3e14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098196.31/warc/00b688e62c0629e58480db2ad8f66b60a29fb3a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098196.31/warc/5edcc67eef63bfd48428c2d57218e129bb2fc2ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098196.31/warc/7477efd1c336f29f220a34766ab002cb14624f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098464.55/warc/1f8ab93110b01dd8dc24caa7ee1b9e1badde4de6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098464.55/warc/210d0c0df0033963f4d5dddbf334827918c1a8d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098464.55/warc/cba5a3b2f7cdb5eab211699211b494721ccf7cd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098468.93/warc/dcd2cf435fe0fd26849c5dfa1ba7cf01c6077e78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098468.93/warc/e84451a3851c6389eeee13a07c302068f5a6698d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098685.20/warc/9ba127c1e6c5faf6e3c9688c5eaa0efd4cd52d05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098685.20/warc/fb19e77e0d79440e7879b56c65e94166099840d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098808.88/warc/373a217e6f4f1901260520bd7611e8b622b742ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098808.88/warc/dcaf99363057fea9274df2a69fe953c726e0cd0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098849.37/warc/1e2561256cea55b261dc581e3ec05f7691397a4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098849.37/warc/be00713862e7abe676e42b44a20ed8bbdab9c80e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098924.1/warc/85cd7e3cc1399e795c3b6c5bb7980cc75b938816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098924.1/warc/9f73f8cfab9b2be1c66744a248fe6953684e9b31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098987.83/warc/0f1916392af4c9012d407b422fcc588e765b5064.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098987.83/warc/c4d5a84ca5b82e66ed20a6743ae5b3d66090ae82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098990.43/warc/09cb761ff679fd0b5087745b5b7b57941721b5ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375098990.43/warc/c840ff1d71a717a4ed87334c1724e06f15f2fe77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099036.81/warc/5cdcc74b5810600269ae080f7d91e1b399324ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099036.81/warc/81f637b09ecfca3c8777652a0435da92d2d297e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099105.15/warc/38da14a9bda59e65568b2425b57dcfb4ff1eccdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099105.15/warc/3fe64e7f5213360af1d3daf9476f8865fe5e9d44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099105.15/warc/aa4febd66ed6da9034af5c25903d9c03b68d9ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099173.16/warc/9b9a1056d77e7c4575a16f17a9eb950b081f55dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099173.16/warc/abfbae6d43081273fad62de1ef310f40172510e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099173.16/warc/cb6264fa3e4ed32e3c2fb612ddd2b6d5e850c7c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099361.57/warc/47f68eb66835cf5563c6a198633243dad06de3d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099361.57/warc/70c99d97968435f236336283bcb005284d73e0f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099755.63/warc/0da7e0d74c135ef9bfb66f384bedda7c96bedf11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099755.63/warc/ab2c96470521a53f5df12d5bc73547b236021c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099758.82/warc/50119068fe9da08cb2dde0c08fdf0581c426e99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375099758.82/warc/942a84934a69a9c9e1c1f86bbdc8d1d5bdece319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375100481.40/warc/cc01dd2aaf8a7db514ed5afaecb8501e82c0e863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375100481.40/warc/e4a31667868903329cc3fa5f2afecc8908a5fb58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375102712.76/warc/03c3ae67f84a13ec33e61047dbd14c7628e0bb73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375102712.76/warc/04abd0771aa8000172dcfb93babcc298988b7c76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375635143.91/warc/146aeb8b1db11c90401fff040c8b98d321b1ff51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375635143.91/warc/da1978e2e6b627338fea5efdb40e0da789c3e31e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375635604.22/warc/2e69c86e875d8a6b3c431dbb8cd24ba727aab389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435375635604.22/warc/e53542cc8935f6ea829ebc1bdeb7d286bde3dad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435376073161.33/warc/bf6b585f7257a07a847611f1b49b9716dbe7ea5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435376073161.33/warc/f9893310337af590694d87ab5403615a60af9695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435376093097.69/warc/22fbabdfbe7861457adb323d03b81acb575ea60b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435376093097.69/warc/3a86a101570e9997b3fd54d9e6937af67b21d674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1435376093097.69/warc/c4dd11d0713d11c122c90043ef44520179a7e34b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/159b00224a11ad11889dbf7f0f2aa64dfd97ef0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1796667638bea4b11648ed2726b4a6ce0bc0d878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/19556971283317b5c8f87fcb9c8359dda458def0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/1afe294708e3b2fa325070fe5a8411c7984542ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/21242b372b96a58713c1a06380cdf4c1062e4d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/21e9ae538dfadbea8a3ea8ad16d20c4d5cf6da94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/2433aa923fd9bd40c4fd09ea1f7e62b1ab532206.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/2e41d8f1535d2c1d789bc057d2825c596e3864e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/30dba691618f3010d9258c0c7f2d7f727aa8e69c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/33808794840aa7b416aef1928c09ed2d98f3df6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/3a29f3997c9facfb7138ea4639ec554a63e2b492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/420eaceb62a0ab5ea91008aae4259446544fee04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/427ba2dd6271cceb6a5f73a42a166ee208ddc190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/4eb32d79f8e7a1e5ec1240e515b9a10df5028e32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/4f2d8fba79cf4dada476c70e713c44c9fd150cf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/4ffb7e98d17235b1d10fc74455dff024ca69ed58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/5166fbdf4fdb7ab6ea621f736f4f8a2b27bc4d30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/52c0647d26aa45c8f42aa75fc8d2a009840cd296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/545b0664dad3477ffa669819f592599bbedbda92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/56ec6458706f829aa417d85d53523e251925d4ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/56fef31053f56e824080ee9e95b1687818f8d0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/57bc20af9c2d251464a5b57b485ec64864786cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/58dceadc6577a408ccf08fd6255a062b14efbdb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/597df5cee554f681ba284f31fb20f8b2d05859f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/5d5600dee511a6c89ce9330b5ab7229f29b20351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/5d5dcdc7fb8fd98791d69bfe8c905e20360c36a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/5f1d63f5d363c5b3a066721b5247589e071cb7bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/5f99f3445527e666b8e38f1b09e9f01fd3484eec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/63c3ac5b7b326720df5e85764b43659d0060b908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/64150726802da39a7c8c6ddf8cbc05cb28e6ba52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/6465374b6d4fced4536b7f78c42b90be98bb6432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/690cfc4de9d7d146fdfe01ec6d130c86cdc9c0b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/6cc432002bdd21eeb1c40bc60c4708ed0f298f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/71d18c37e0f72091fe22996cb494b2c3f9572f46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/75c35af07096a2459fc9254dbb555bb3e318880a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/7623a1370193d6294637f2876febf2663a4a4c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/7d9537d3952446d137ec5f256214ee8247e26588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/7e29cbc0fb26b096c23ebf6c67a9b1e764bab633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/7eb1c26d848960dd9379f29a7f7674609aaa3540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/808e8aef8b606e86cc73d8cc8ffae598a18c13ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/8102c0fc412b4cce9e814c1c562bae6dbb276c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/8179bd22262e01408dce5a3bdb85e6c7f2c91fa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/83f6021a49f6496df326188bad40932c729b4693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/84409d7ec9e90fb82f72b36991cdbd7ebd320d15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/8906b08694302b5f6ded43304b4d69fdec402246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/8b67d3be306f7f29d508baf5220bb554c2a0f13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/91056cca751c751336fc525c2f850a1c9dc6c252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/92d7d5c1c4536fee5351047995555b44c508e8e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/940b0e4294a27c25778e11b52fbe7239ee7defe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/96fd0b3267ab6a1b0c7f0afb14d2c01601596003.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/978ca04d27ca0c7b93a8881e9c1c6214abc9b04a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/999003b88395f4129dd899fdd48d1d0798f67b71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/9e57f44c30fa945b58fa2bba8fd9eb8807d50940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/9f3f3258c374a050ea2377ff2eca968049a4c9a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/a561a748277dc491c11ea8723fdac1e03ea2970d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/a9aec25ea8bf7dd36cc2685a545dd17a639c0dc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/ab1e113ee81d5a89e634f334f20aa481ba296a90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/b4fcdf528bd0db6f93914539acb8012aace5d331.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/b56c198784a149c1f5ad5756a6d9eb0c0f6b3359.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/b5b27be865cb33929627125072b9d85826de1148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/b7c5bd4e5458ec2cab79e912b44fe410d7e92157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c10133d024fd87d1e081d8ec8789e8d77eac8247.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c16de52a8c3e4dc074e4016c5ba2547553ecb163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c57c361fb80b283af1a7a40e76db4152a1075a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c5a166539f9083982d733797f80a5a143601e8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c7c0987913e1ef903a09c2db11f3823418192469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c813a105acddd8099c8c9281509664bbfac84ff7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/c9c651ec19091c2a56fb11775dbae34f8597b2b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/cb5efe8fdf35dd4276a250e06c5577caada28312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/d042032e8d0cf199b06db1c0e77e4a086ef3b284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/d08d964b8252afdbd6e94aa2bb9949d177eec458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/d234567f223b3380324932f63ea81470101e58da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/d38b68cba5070079ac21b77244c5e1ebd3265059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/dca855220269cf4836b5ef0576676fc932c6ca36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/e1f5fdae59ec5c7f571a675ae58c53ba1bc7d681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/e22059ae5c905e05ea19cf68c63c9b0943e63259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/e899860a757736a9e240b69c8cd44bad0d706357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/e90118b5b929931e8e66142330fc8fb81b578455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/ed2273c294f8abf3b1dfd577bc11347986fef290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/f1cdae317658a2a10f4a4cc972cffc244f6aceaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/f1dba3544b954e8355e8d9be9af7d82f65acebf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/f513e5e478b5ecd33a37b1b7690544c3a940e025.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/f7d79b91d5c0c8f4cf1fac98ccdbae48130e1f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/f96075d032101100d1a1fac6ac6554e8d96b9a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/fd22b5ced5501e980b6a9fb69361b66f7301bb91.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2015-27 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-27/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2015-27/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2016-07.yaml b/configs/cccc/mixer/CC-MAIN-2016-07.yaml new file mode 100644 index 00000000..07149ed8 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2016-07.yaml @@ -0,0 +1,463 @@ + +streams: + - name: cccc-CC-MAIN-2016-07 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/05f49aae3ae4206afe0193e7c6c1f5eb5d112733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/064081efd7670ca6d8270fa56b89e1e8205b1eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/06902ebde9305237e3e39803a03ae5ed0e9473d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0a12ece6f5e89b18d99821002d65ebc3918bb060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0ca988a9f200a80b9f79052e757f94a5c8affe17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0d53b9b2e28fff9b958e519db1643ff398340a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0e45bed3bd845cadf205294b8fe4f517fcc439ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0f78c09c93ad9177b48be731c813444861350f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/0fdd56c8446490505aa54baa27b30141bb26b330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/123fe99c70788cce3f2d1eb0cdbad3fd1772f358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/139bf949e20308e3919c98e6fbf52aa5291bac8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145519.33/warc/0c9cfbfeed95b91d89d28ec710c497cc982a0be3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145519.33/warc/27ff9c5087371a0de8af11ed0fbf1ee898a41cc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145519.33/warc/5c847d023653cda6213283e1ec6c02dca54ead7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145578.23/warc/633503d0ffd99c2547a2b5293d4779fc800329f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145578.23/warc/d042b5b81d40b410d2205b604226b5c2114ec524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145751.1/warc/57c66e3da7e726da1fd2f7bba7683afb1eac5442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145751.1/warc/c01936760af04e8085fbce26cd2f11afb5b4653d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701145751.1/warc/d127d897f951f7c0ffcfbb9cce611fe8633ea25f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146196.88/warc/357d695b297338ebfc82a886d0f4c9ac2a6d3ce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146196.88/warc/b0d7072067f1b3e85ae2966b55fd7532cff2fa1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146241.46/warc/29848f41b9b59880e62159baf69df8ac60e5bb26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146241.46/warc/a264bfbec76aaccb2902bd5ec0871ba8498a96d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146241.46/warc/f3f0f89f6d0d2e07d047b65ad80245f7deadfbe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146302.25/warc/42a329265bbf2a5390700444702e791d9d5fd4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146302.25/warc/56f2dd36ca87c018faf203c709a59da731f2e72d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146550.16/warc/53a3cc5e4bc8d5da21638ec6a47ef0bf5388cee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146550.16/warc/bfeb97340fa34ae0468ce6e3c1246c4568049956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146550.16/warc/d4d4dba51a04c3679e0c7ad2ebacc3df79b59630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146600.56/warc/131ea725c91fe38c912da4019773cfc139e2896d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701146600.56/warc/c4a73c808bfeac512e3ac21442934a3fbd74a7d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701147492.21/warc/144da67a06f91c03484fb366b4584f79e12c7bf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701147492.21/warc/59ab1754314cbda26a96b6876c3ed9378d1d5ee5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701147492.21/warc/799f7b38e376f491559f163f28eedba73e4a228d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701147841.50/warc/3e5446a4b271c2810cb30da4881de9d3472faba1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701147841.50/warc/a8a593617a9b121290cd164260313eec198f56d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148402.62/warc/10109f2eb5d6ddbd6b4063c0ef35c9954d803840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148402.62/warc/1353a097034ec0bbf4836414542bf1681296399f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148402.62/warc/4d8e597393d4101f607923541e9bc63547bde01e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148428.26/warc/69559f775ec05d72c4ed2b1903237cc0b0e1222f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148428.26/warc/e39e746e63a791bd1287bea5c0050fef04505404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148475.34/warc/0ec4af80b9f6af7eedcce46fb06b301f9688456f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148475.34/warc/65f1ff5a5a18edee949e2f8a1963b5acfcf8797a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148475.34/warc/b6378e81d53e168738157c90ff81324f97fc58d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148558.5/warc/39ed579a3304987913b605cfc81b4d41e83e929b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148558.5/warc/42af8723635852ec616208cac97a97a8b6b8868e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148758.73/warc/24ec3dc23dc6d014a4e76d15ca1d9ab25a2d14cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148758.73/warc/73a55a764efedb25d965a3183e7a862b1b825c12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148758.73/warc/e6fa67974f0c5d5b8f7926a0fe520e6360d3f362.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148834.81/warc/b23545dac38b6a44f57438ec66983e0e2a61607a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701148834.81/warc/f9a9a56f5f5468d7524e51da7ec139923d8b6d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701149377.17/warc/4140fec128d1e64f51b7920a89648c6f54220d98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701149377.17/warc/9cfe29b82a3de7b0ae7532f4098fb341d7793328.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701149377.17/warc/db06395c100fdbd5056930418492a769e5b266da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701149548.13/warc/6e0ddcab872a4fa9319e38005f57e11d7b8db55c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701149548.13/warc/7bc4ca9ce76bcd396bc9b6e845440b9bbe158ef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701150206.8/warc/85d4891855910f6507be61c2648f1c24bbcdd65a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701150206.8/warc/8ecc319eb05f137b5b661e66fe75fd4202e18167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701150206.8/warc/e88ee20c03595bd20760a71251fe57a6459e7394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701151789.24/warc/bfce9b1db8d254089c356b28c6fdad24a1ee5473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701151789.24/warc/d6f8e3249a6fc27164dd164f78e6cedeef2c6d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701151880.99/warc/0ff1de7416b8303d85347461fe1bd301732a77dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701151880.99/warc/445665529b676edb82b4a6a263c5126cc82b2e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701151880.99/warc/eeb3fb5d90b3eaa81d70627d4ec91ebe6f06bc6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152097.59/warc/b228920c7747f144e788be56e0f73969f4066d9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152097.59/warc/d21d8ee065bd6d94b6ac1519781bcf9e14f5dd2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152130.53/warc/34e69ae66548d5386a18bc8c49c0a5ee14d8f37c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152130.53/warc/3b91c64195a0f935184071566b6e3257bd2571e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152130.53/warc/c778df90ec7f66da3a917de0ee8a87ac03c97674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152959.66/warc/1a9f42690d598877acd59889cfb6541e5b2ffa46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152959.66/warc/87e0ba77de9912a5fcdee24b38e6c5c89883589c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152982.47/warc/10b389af81831800da60c12bf71e0d76a736093b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152982.47/warc/63c964ff103781a0837ee2d1bd29ae50b8fc5d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152982.47/warc/ce701c15e63d063c2e0b302b47303f11e178fdb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152987.97/warc/464919e1f19e10054ae265b4553754466c9c0375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701152987.97/warc/b63b0282d02584e0f80e3850efdd5d1d93b89326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153323.32/warc/0a1ac9b23607d9f60998f9fa9a5f8dff7ff71d75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153323.32/warc/7b66ad89003f9499fcdeee2b16e1737f3ef21c84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153323.32/warc/bf1bcea9c0117a9906145c4cac6c2f09d67e9e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153585.76/warc/a5cf0110bf5763a6e8d75e08c801b16e85623995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153585.76/warc/fbccfe5f56f309aa63bb981ee919940cc1b97201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153736.68/warc/27b5b9add706f2d7d3fefaf80a8dc3ef2f53870d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153736.68/warc/4590de27ba83b071bf7da12c3d2c4304bd9c8922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153736.68/warc/d1f1a71ab7bd97b522b1504bd0cb35bce7c025af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153998.27/warc/01af416b5606aba1fc73b5e8df71fc621f684233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701153998.27/warc/c9a9289e6f23b0fdc95075ae5dc378f9b5329705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701154221.36/warc/2df10fe2c8456f7439895b1491e9772061440436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701154221.36/warc/c7a1ae0a3e78fec4943cce4f491f5d3f929f7f52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701154221.36/warc/f4e479411b644773ab18f6a86855b70deb61f30e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701154682.35/warc/3957e1fb1be868585cb8817a62a5a4e797b0df53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701154682.35/warc/497139004549e080e359313a5f5309b6d7f08698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701155060.45/warc/377fade5009381081745dfd3ddde2d30e8de9b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701155060.45/warc/590a58927f484a339b306bfb992f1ab91df27602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701155060.45/warc/f6fdd936f64229edf4931ecc8026c2d60095dc0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156448.92/warc/7d1441922d0b41e28b0c7de15bfbc7cfbf1cb1fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156448.92/warc/81c4295ef3523c3628c0ea317f2e2f7296bd69ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156520.89/warc/150c35efaa115a4936878ce60b4aeb176ae13b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156520.89/warc/a6e401cd0ab07c63319550413685ac69320dad42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156520.89/warc/bf20a6a8a8beff611a382eb57296c210b468767b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156627.12/warc/be50165ce532689b1d1ebd3421f1ff259b53dc59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701156627.12/warc/c7c663cf6bca99e5ca236220d5742a631408cb9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157012.30/warc/3aff5100f3a4b2957211fbf88873212e93479c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157012.30/warc/bbf65610cecba12d73350955df59f4a5258f8c67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157012.30/warc/c3db3832ec071fc55b85e5663c43114c8d7ca7df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157075.54/warc/1e32471d807f36ab5576adefdf6f7e0011b36962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157075.54/warc/2b7790370653c76607dd5fa6f42ddb10e6c6a60f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157212.22/warc/31c078cd4838af3a26b788145ae19850902ce0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157212.22/warc/7b355b7f93638db673d08fe141c89e9de8ce8f5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157212.22/warc/9dbc7e67969da51e48fb23badaeb3e5934815f96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157262.85/warc/2a0bf4510d1485be57acd6cf1b1638cb58d1410c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157262.85/warc/cb4b9bebbcb3ef28da5fd4bd1113509baf9cda9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157443.43/warc/0605a180dbd184347277840cf845546f34c3728b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157443.43/warc/09875d18fedbeaae76777c3ade7a60927cf496a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157443.43/warc/1aa9eaa29419b8c99b1bad0b33d37f6d014c5f20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157472.18/warc/681de7e2cd20f83892f785d31ac287d19124f295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701157472.18/warc/c76c670959ddd01c7c0809e410926bbb707ec908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158130.6/warc/2f5291421c3917766a17b08b93c42a1f47344906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158130.6/warc/717b2f04aa5c34236ca3a111d25f86993829304c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158130.6/warc/ca4ea0136a08bb95215ff12525ab87e8df4b53d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158481.37/warc/429aca4705b01187036965d379cf0555d2afb5c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158481.37/warc/d84d50910db0e616f9a638366b0770f11bc22f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158601.61/warc/274fce17f77e82a90914f1f06ec5cc29b528147b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158601.61/warc/d26f63891a0f5eea62880e5c680d5aa54ebf5b89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158601.61/warc/d3049daa2d097fa3a705a2077cebc072b07f1f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158609.98/warc/1a793c4a332f1b4befd9be26013546b273f8497d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158609.98/warc/1b66c07a9b954413da2123276db5f57cf0ec0df9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158811.82/warc/9e5a46486335d23faf6e2dcdc9858a46f9ae1cb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158811.82/warc/c1d603aeed443d6605ad8cb64517440ed2f4ba8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701158811.82/warc/eed3d0a41f131194c39ee2babccaf41f284a01d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159031.19/warc/4e249965ed1e1c9da9b72f54326e7b900bad2815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159031.19/warc/b609b5aaedce961a351bb77f6f6f94c6b48ac4f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159155.63/warc/467ebfe115fe30fbc7cdf7aedede9acb1c61cb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159155.63/warc/4e4465fd5a38bedf4e76e4896d31a5eaa80d9447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159155.63/warc/84d2aee06eca22ebf97b55e2191e228248b7a9d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159376.39/warc/6f87e9ac3a7139311eccc22cfa5372540fae3c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159376.39/warc/bd6b27f6931bb6210d8277e5c34cbd7fa2a40266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159654.65/warc/a2636da01616caa2b5c48bdd224cc8391f4b1004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159654.65/warc/b5c82e05e159f0cb268c090bf5938f0823dea506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159985.29/warc/35e10050fdd597addef98a69e8f1825e498ac212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159985.29/warc/a91810e331505c97d3fdfcf6f2eedb05ce3dba59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701159985.29/warc/ef0fcc27e310fc82f55e5ef4425d3abbee188ef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160582.7/warc/7c59bb6a906042823d5ed498b3c8c03623d6539a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160582.7/warc/e7234f2e1ec670119de157fff602500b9eadf5a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160822.87/warc/0b0c0a66ac679a799cc95d439da32595ddeea6f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160822.87/warc/627e6799bcc43f8a6335f820ddc0e5ee1e7cadc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160822.87/warc/d7d2b877d0c3062f143d7f4b7534e309a55cbd93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160918.28/warc/4cf863c9ef619b360f94907c91189bef53711a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160918.28/warc/cdfd1ac3cfb141d11340f4758eb7888ffd358a45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160950.71/warc/3438dff1d75e00fe4402fc7e4aa66b515bf158d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160950.71/warc/51b195ad1f84aeceb51abd7b91768d630981ff9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160950.71/warc/6b5ba1677c5e4ee9566b1569f83f7c2cb7981790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160958.15/warc/1abd6f9e295ee8e77650fcdcafae2f2b856b4a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701160958.15/warc/7b39080694c441bccf1d440d3e7a47a144fae3f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161718.0/warc/539e0936929b79549f34b9fad1925a853f940dfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161718.0/warc/bd4a0a8336f053aaf924534dc36fa091a9da4dd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161718.0/warc/ed521373fb3378ace9bd22269c36f990ca2e7850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161775.86/warc/c2bd0380a0b22060630a4de76c20d7fa19c145e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161775.86/warc/ceb12668a64a0aa212b784c313f34cb6611285b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161942.67/warc/039182aa90fde08bea61da1ae1b964fcf90acbf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161942.67/warc/2d7c6dd318d83d40e393b88e8344aad41703fb82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161942.67/warc/3f3d9c30dd1595741a9812531078585fc6f12bf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161946.96/warc/5badc4026d2375520e7ef30cbaf0cbd645ef4dc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701161946.96/warc/822baf6786b0feb152f7e3aa79c943a16c534091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162035.80/warc/525e9612b0820efb86f98065722ec8c99da5914a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162035.80/warc/a99fad18e2df55410959db7fe45742a04ee0dad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162035.80/warc/d526113fea936bc6f8f2d92bebe72ae4de357c9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162094.74/warc/abc6b9fd84be82ac57a6fed81bb0dda660d5d000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162094.74/warc/dac16dfe234065e6c58a2324cc22cebc9a7d0c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162648.4/warc/1ec7aa08314d44bfefe3ee107784f4c71effdc4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162648.4/warc/21c0fc1a5702c9657fc496b5684e7fa2cabb1332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162648.4/warc/4baf532edc230a069532ab8beddbd061803cd895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162808.51/warc/596a911beca1e2ddfe9ce32df17faf856b811a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162808.51/warc/efb9ab9566c29dbace2a8323ff731eccbb10c7aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162903.38/warc/0d21b2401fe43fe77e10f0480decabaa586ebc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162903.38/warc/b0514761a8dcb611ad9bc6c5262529b850624fc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162903.38/warc/dea51a2a05990c471c122435768e59902c295cc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162938.42/warc/8a0d560e617cea0cb9463ead6908b90347498f52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701162938.42/warc/e75885e1bfc02871a63ee9691fbaaa40ae74d7b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163421.31/warc/1050c6ef24976d411223caa0b6d47c6761a4b8a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163421.31/warc/45dc4d681f664458f3990447184009a821c27392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163421.31/warc/e783d5a941f5409adbe7fa184cf2cfab91a96614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163438.83/warc/5a8113f236889d20067e446ee545ba3209f8bb9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163438.83/warc/cd5acabf28f20b9b01e27fe2650916525311a757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163512.72/warc/4bec65625c7d5498cf0b37cbd4b22722555f4c36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163512.72/warc/797db8be4ea29c1706142e5df339634a0e47b239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163512.72/warc/c25d96cee30e13553713e53992ee98fa78665970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163663.52/warc/473d47411333157c0873de9b5f357d2e367939ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163663.52/warc/62fae6ad0990f3155fa4cb1d085e46999c21f1da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163729.14/warc/1d7e2669b681f0edfce4cda3dc1922b7cb14f68a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163729.14/warc/72386b0049ed4bfbe9517b3432fb01891e702dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701163729.14/warc/adc30da79a737d554211e9ac7bf8b97ada3a8886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701164268.69/warc/c32d006f03430aaca6ccb0d94919035df95f26c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701164268.69/warc/d2b92d6836ef6217717ef85057c53284fd4c0011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701164289.84/warc/08fa41142f80f9c37c0c957d5299847821a9b862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701164289.84/warc/7067621889cb387c3a4b25515a0d2efb9b197abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701164289.84/warc/cdead2e1af1efeb8cdbf8823bf353a3547d181e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165070.40/warc/4d42140548f2a2264cfa51c56410c8586819dd98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165070.40/warc/d099931fb551bfc51341e3fa8700900645b082db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165302.57/warc/15d79efd894c802ada3d454b51cb57fff3a17fc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165302.57/warc/63650a69f9f03fa42cbec8cada9eb4e11f6d349e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165302.57/warc/a32ee402832cf4e94df89d00b586ccb7452f9e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165378.58/warc/56b1e41888abf882e3cd4212c70000578fab4a1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165378.58/warc/7751bb8b84893aa0203858391c4ee8065eb28e94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165484.60/warc/0eeedf26a6ee1eed8157cafa9227a26b76747c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165484.60/warc/a11ad7858b199fb84f9eeb6208d4891b1f0e280a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165484.60/warc/e71b4712e7068c05f7b053256bcfef35ee62c3cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165697.9/warc/37edcdb3bb1970699f18d4e190141535f4b9d1ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701165697.9/warc/d9e4da13c73cb2bcd48e718b559619cb9279829b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166141.55/warc/13316379c80a28b548668bcb59afbeda703deceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166141.55/warc/6e153decdaa7aa505c962fc5bf3fc1081912007b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166141.55/warc/b48a555cba0f4d8cbd65fe9c651c0e0df0224632.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166222.10/warc/51325c2f7c0d1d082412ee2de81d7de46b9f0318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166222.10/warc/fa223e1a33a3111bee08ce3f062c5adb3601ebe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166261.11/warc/6f99e3d44a3cb64195d795f4c91f2c384ae45303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166261.11/warc/dc953bb4c19c86ae35093de3b356209988f3fdba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166261.11/warc/ff8205af948e70ab27d6f4aa689e355e54c93a66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166570.91/warc/9cdd4a304654561408d2f078cc1f64056b796c5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166570.91/warc/cb2e8aab899ea991bd05d2e7ce85ef209f6326e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166650.78/warc/2c6835f4113768dbbdc681a4ca18eea76d964601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166650.78/warc/37bff513c1bc0b03b4b2c8a7985634850b00861f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166650.78/warc/6f20aec3a0bdd67b5ee98771ce840bae07414350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166739.77/warc/34f5924ea4fecccee99c5469d7facd1afd0031be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701166739.77/warc/f140382d06a5ed387162a3da5e1eaaccf9f8d9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701167113.3/warc/06f5f4265d87c0d9de0992090b948e8a03ad9b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701167113.3/warc/09dbf45cae5c25b0812544acc87bb8c92ad3a6b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701167113.3/warc/e35b0fb0f89627c2ce257e228eb3ed08f7677960.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701167599.48/warc/082a3bebbcfbfa0baa96319e7c1774aa87b0d81f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701167599.48/warc/86186d2c58e8954aafa81b1a5dd07e0545c44c14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168011.94/warc/373d4f4286ac5fb187a12e3cf6e01449d66b8c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168011.94/warc/c982d3778ff8c655080096d0615c3a279668e663.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168011.94/warc/e041d10d4fc20aaf54edf20bf91ca55c4b1464dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168065.93/warc/3d9f5b78c57be743909eab609627f68c8aa7b74e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168065.93/warc/75a1902bcb7376680e94453cfe44e82d0e34ad8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168076.20/warc/19c8c545c8596bb53889673a9b4c0cdf49598ded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168076.20/warc/6fdf5eb14022c9521e17b0c2e94cd28315f2d72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168076.20/warc/8063ce1bb2d483729d4a23b2e3d9b943b2dc9a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168998.49/warc/53f33fb792debe7081eb53196ebe5daa76489fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701168998.49/warc/aa984219440f82eabc6afbf480eaed96c972a9f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701169261.41/warc/1b4247c48de50dfd31421c4f2f0865f786efdbb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701169261.41/warc/86adec9b94638c57474c76c379799d425f020a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701169261.41/warc/8fa6453ecdf7d8ace8119b4150419a094427f7df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701171770.2/warc/1f557f7204c3f503f5eab05b78d3af9b26495b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701171770.2/warc/9b8a772a0f4619a26325a5b97a23933085165786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701174607.44/warc/65cea208ea7b8b091df680ccabf28e41f31c7c15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701174607.44/warc/664e1fda019ebd489b57c3eb09c360f3aa6fb6f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701174607.44/warc/fc6aa79272195bd96518b45a076fbc6a612089b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701962902.70/warc/8ceab798c417ae67ee41ca1b174947716c8f97f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701962902.70/warc/a81d3b2fdbcc62a3d1445d15331c67d805a68515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701987329.64/warc/86690b79f6400f2161572c26b1d203c49ffb9062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701987329.64/warc/92a4150e93c2ed886e17f8e63d7695e8f8037842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701987329.64/warc/ef0d1d52318f77e5342f15c306a0bb57df5462fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701999715.75/warc/8bb54d6e26ed0578abcef1c9a2c94d909f720c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454701999715.75/warc/977686eaf4465583e5dbed6d670c6bd5ac067392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702018134.95/warc/c8368ca9ac0c33345f6d3b38e29d39bfdd33b4b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702018134.95/warc/e9ad39de83027fee0851caf04c84c47cf467a22c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702018134.95/warc/f14123727d7f7bcb26f6361c91633b0d53e268e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702032759.79/warc/50fa8414e09842890981e633972bd665d47721f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702032759.79/warc/88f8080762b4229c78ae4a6d5483c98c00f6b6b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702039825.90/warc/08b9844bdeb0fb68a5de78dc9fda0cc2252fa161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702039825.90/warc/5944b476f470114faf3b3ad740d90e94ead7c2e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1454702039825.90/warc/a13ffd24140c17ef6a5b31b6e8ca8cc8126da0c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/159e2a8318cda60a707f30882e70dea63e01f8f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/16e91786248c38aff1bdca8b7cbb0a045ec47b52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1a5dfe2facfa03de48baca6198081097ac591960.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1b1fbc90370b2e8e45b29e4e2b9029b190570dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1c0c1b6fcb1780654b127044398a21647345eb82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/1cb7ee84a4ff94b8fe7621b116cc7eae6b58036e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/2189c43bf4994407824030905b5513c2eb177646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/23d22e256c1b48c37f73a3c5e060d944a9dd3bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/28e5b583d0e16ac8c4f66dc50d3b69740dfe57ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/2b864bec459c6f027407bbb0a7bed5281e1afa5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/2c3e7bf0551e9c8f18caa9eab15f4c1585873158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/2dffc8c12542622f9ed68addbf2dc2ebf3b1efbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/2ec086199474cbc3732d5450ec28f31b691dae9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/30aebeec480c86297015e28a9d0cb21df85f60b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/316de4722af6a278c6471248acc8248d2eaa99aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/3408840ad51e0fb1cbbcff4dc2c4e5d47e5717e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/393fc2c5aec8cf71f76ec9ed3617af3b22b8aa98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/43a0ef3c88c831a2e1b39561f5bebddcc5a60831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/45061ff3873c946fd18cbc63fc3e9c860172f604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/4a2451505a65ecfb544f07f274a541ee9026396f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/4ca07bb59944223662034c117a563cbdf6879329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/4cccd8e70896d9462aa99d84dd36beec14d80e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/4d4f1932897392927386a60eab5f395392e12bed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/4f295088584ca2590c7d93032d9959f5944b890d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/51f794e0204d40f4e1298163dc2754fb8d45db40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/52eb30c87994d5dc857098377631da52c6daa0df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/56452c88a222ff11c58cf8c03b5a3e8d62dd052c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/57edcedc7a5a1c24c40e6aa17e1b69a1c9e6f246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/608e8f896c926c9dbcb8a65cf8aae8468ed271d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/63bd0f1f25d678b3b3e28977bf922dcdc7f88033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/66288724e0e4b5c0c6cd1971fe1172e43bb45a3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/7032b271bd45285a057ebbf0349b8dd2eb90fe5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/71138b1cf457d9ad9e418a271a77842d9b475702.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/729e7b8c1e96e710752d4f43c2cba80432c10235.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/746a192f9a4ad8a12339b8540a54e41641ab6126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/77569ef69b27adfa5943100b5d5adcbe71373eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/7765b99c7130c61f8dcb8e0ddd73a74c10d8e07e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/77e1f29fc193f91fba83a5af53c1f4c53b610b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/7becb8adc5a96d516dbee44f4ca157b9d0040139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/7fc36465fac0124bc90bf98aa2d93beef581f59b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/80218e35d2e91aab41554927bf8356b4c1f0495a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/81f41eda906657a0004573a4d926da6cd52edb46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/8e6125270f0186bce58f765b3864952952d73801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/906b91706e18da2d4b6d0c440e4745b09e8770f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/918bcd083f6b038bc630f1f4bb4ef9378b7c3ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/95fe23ae49caf3bd01b07c30733f436c671381a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/96939a91f09791f429f8bef75f801f9e006e0ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/9853d6f53367eab0564198b56b50d74604958a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/9bd38740e4616bde8b6eaab8666405518a9133b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/a32f94e1fb9e13cb12341141a6a73d7aed4139ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/a369929c64ddc35c5359f9feffc14c5049ccf0e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/a45146322330d6a4b1cb2e1595a397bc4aeb454e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/a65cd38caffea392a4a653acdbbe4c2b2d3bff03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ab1dfd683c5b5ce067fffde44cc889ede3529b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ab4384863c8223a0a3b38bbe1e42f28139eee2f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ab529ab455f2cf9cd4d5e03a4bb1f724b0a9899c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ab91bba7e34e74b98d1efe5b81b89fd7ed4eae57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/b2b7a0247a99abb720a6a5a20b15edfb511a4919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/b50bf1020d699d4910d884b9d16fd6db0eb293de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/b6499c548a11e0ce70faed1786639f04982a1dde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/b71ea075b2e194494a97b372a8b5b5cebab0f92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/bb0d087ffcfd10bb9b07ab7a8ffcfb00d8fd5c9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/bbc08f3a5bf38b99116d1333f71ebfe39023a0ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/bf8cc9e21b16b5a06815cd1ab37e9c81de214f13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/c0ba8f00d2ce6633a93ae778926bb58831c85dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/c663513b0c0be4fe80f7b13ef6a232ac4dcf4e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/c76d80cfad7e513c34a1ebc4b3debc135749daff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/cc250c7bc41f3edd2e6a068c69849de2b802af60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ce4f80f1a9fad285b01184a248982b8c15a40765.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ceaa68aba086fb105f5d945d0313c956fb3ed020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d2b1d803a75f6336af72a1bf101b36cbc9a09608.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d2b4e85ad7a861d61944fd9a5cb19e8f6bd2c89f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d593ded4ef3481fac10fa845c03bfc486c2d41e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d61e0e5241509ebf1bcbaaa20ae363f6670c13d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d81c07d98b3ceac2e7688a8b4565c17b49aae813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d9174dcb5e638761b43fe8f572ed4e97fe6e2137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/d941983d0aa819caa770f8f964eac938b6a9a948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/daa6c6d2f410ea748a2e014855a274ffaa2e7698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/dad2c6bcca5e7d29b72fa44f9b47f26be1b1b509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/df44b2e4848666965450e936a8f0ffa5d59a8da2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ea10339d0a7ca362a80ce2785a7b35d2a5e50d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/ed623442ed4c0b9579e0ff0433c4c4b7a8f03780.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/efb24d28794ac99225f847fddfd9a3120cf81042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/f776e4d097914493e3a1fb916b48f9eb8604dbf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/f8364ca6ced44c98b2965a830e61c12ea320e95a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/fc98243c31151afaf8edd4749e8e81050c8deb55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/fcd0b756fa173ea7ee426b6ed117cce9b0f56ab9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/feb185b9405ed185fb257f19611919394226d044.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-07 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2016-07/mix/input" + output: "/tmp/cccc-CC-MAIN-2016-07/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2016-18.yaml b/configs/cccc/mixer/CC-MAIN-2016-18.yaml new file mode 100644 index 00000000..464b1e7f --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2016-18.yaml @@ -0,0 +1,336 @@ + +streams: + - name: cccc-CC-MAIN-2016-18 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/02ad2ed1fee224f791c9c549bc74fdd92d824be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/04fefc4f72c4b669d1ba7d4aebec224462863908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/07933d9fc10af51b0b2256183b1c6173d970c02a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/07d8569d33b639ebc3b7ed685bb2f14abf73a2df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/08f2a8fa13d3a2954c948becdd637982e213fa8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/0b0cf0d54236cc16cb0d30b39bed205cfa6f6516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/0ea165be481a26d0aaef7a55bf3fe46f9b5794bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1029998319bca8ae738420773f74755ca94a5ea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/12f3794e76218236d4c4bba5ff20d88b652edb3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/137758fef2a2a744b0214b24259a5b20a3fd1077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/138f0a132dc1c2b7d92d75dd4bc63743fafbca8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860106452.21/warc/649256234c09ee8791a63bfa7443ee74fe5d27db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860106452.21/warc/f07e8f6dc32f1ef9e6cc272fb55cf6f9a3f38129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860109830.69/warc/be467d8cf1a8f447389498b3e82af65f2e6b1947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860109993.2/warc/7dc0af9cd18fa826f05e27482f9ae182e37e3537.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860110356.23/warc/6ab8253e611d85b557c038c7ccf239bc5b214d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860110372.12/warc/863728089fce0ccd231d4e97d3b3eaff42944a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860110372.12/warc/a0451436448ff7526fbb6a54ee615e30fb5208fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860110764.59/warc/8771fbb14e98f0dbb5277e055d33e94549156a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860110805.57/warc/0178ba1e5b03c720fb27e01152e1fcbeb03df806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111313.83/warc/e23a8dcec446b9b7a5ddd6729c3ee3fee351572a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111324.43/warc/00436312003d5cd37eeb505592a47652f6841aa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111365.36/warc/4028eafbf1e35c44242f3b82749c75d70e62c7bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111365.36/warc/62560e5e08998938947358b08045285b843e6bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111374.13/warc/9ea9ba9c86e48d4d6166946433c7ecce81f5b5ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111392.88/warc/ffc23555ddc6f4b6a99808b228d0a245b92ed5d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111396.55/warc/95fbacee85f8cb7e3ca0b5467a5f1fcdb41a340c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111455.18/warc/6bf54c60dddf8114f13e9f0b0bd77d6479eea549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111455.18/warc/bbe84390f2387a27a977202455aa02aa01f02a0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111518.82/warc/2ee7f97a58ebb9bce3d875fbae50ec24af67fcde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111581.11/warc/98edf6abbef21723168f4440c8bbb921e039d429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111592.84/warc/3a428efc2a902db87b54f4041c93e55a3de6289e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111612.51/warc/68e0eee0321e2bfacaeab5c43a06d30cadf3d6f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111620.85/warc/9a569b971308f3aaa9f5052a01f45ad4bf134916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111620.85/warc/c3a6058002c35ab203a08041aaab0b950f8a9201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111809.10/warc/0375f7d95ce59ab06fb10385714308c466967976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111838.20/warc/b1b05f2ff195e795182aab0319a56ea77197d3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111865.15/warc/673e637394b23875d0fc18a040d7c12533d848b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111868.79/warc/9c40f1a45553172bed3fc23e9442d37db4e1c6f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860111868.79/warc/d5d2c614140dcdcfe36423caeae76012ae5dfb56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860112228.39/warc/ecd6ae672325c27041b986ef242bc5b6bb30713a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860112231.78/warc/5266a6fbd1843bb028cce23d7621796e777a8b82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860112727.96/warc/d4327c703c0ab8932f1feb97b9ee3975dd8ba7d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860113010.58/warc/0e3d1e57d43907a51a1b3cdd88a37fbce3c5d277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860113541.87/warc/cbdf06d5f2a17b37c83b4b2adb8d8f21c1810e27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860113541.87/warc/d22090597533e91aff4c4eb387fa42df676b8a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860113553.63/warc/bec71eb14ac9e3ae0bd66a4e6ab40d0a6056ab9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860114285.32/warc/eaa925efac0318228e203a7e893d0a08760ed183.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860114285.77/warc/328e304c0dd5134fc9c98d03840fbfda008a0979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860114649.41/warc/0c6e857717677f0cea92e26a07b647866504d5ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860114649.41/warc/9464393a00955c5b27013dd4aa3275a69489aac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860115672.72/warc/d0154a3cdaec18710e58d4e4ef5484c6d69ad43f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860115836.8/warc/105ac07d397e8c94a2b5dcc7f24eb354837b8170.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116173.76/warc/f77c3062dd609499adf9dae9b80a2cef3ebe8c31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116587.6/warc/6a9eca521252000e1f5c9d13b3c3300de4ffd850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116878.73/warc/29261769804d5db8894220743fcdb0a5d8b975dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116878.73/warc/fe40f5ec3a4496a0a6e7879df0de158bef3b34e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116886.38/warc/df2d3acffcb2782f6b73ee4076f8cca7ac020f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860116929.30/warc/3088446af1e073b7a8aab22c35f71d16499fb493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860117244.29/warc/8f21b904c5db624319af2112a9f0ace564c6414f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860117405.91/warc/72ed9e0237e6be98eebfdaadd99b4df2d3242747.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860117405.91/warc/b177d0d4a95248e7d75d8fcbdecb8aacfab97282.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860117783.16/warc/f7f8891ca3dabb7c1036f07833238a756dc85c1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860117914.56/warc/a60e4f28162bc18bab3d41181ad922a98f344575.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860118321.95/warc/5e79ea63e026259fac39c38931aa373a14e7ea92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860118369.35/warc/867643938b126918e9dd1c2c4bfc163b02b67b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860118790.25/warc/4815a368c94544765c9e1320519ff8fbf7f7b6be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860118790.25/warc/b2bb50d5cadc56ad302ba505eded272a6064cec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860118807.54/warc/059ca2c3307bd469a32354a49e92f0b660cfa797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121090.75/warc/636897e65abdae48db89700747e65e15efba8aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121418.67/warc/475e0840a0e776b3f4855a408b7be0c1d820fea1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121423.81/warc/7e23cbb22636ab877b99e1df41fd4202b5ee41f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121423.81/warc/d12d89ff60cd03626d0492aadb4026e07c173b1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121534.33/warc/5ef6bc1433cc4c39db6e728bad06b9b80ffd54cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121534.33/warc/9a74f55a1725f7dcf2ee4637e8245072314a88b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121561.0/warc/909c18563af208d28e7fb20fbfebf8cebb55e432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121618.46/warc/c56ea4f8bb22d49c3d5be49cb6a4dff259c368f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121737.31/warc/2c6c2cebaf409ab10bb2da32d63299c717ca6557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121776.48/warc/3f89560949636cf58f3d24f25aeaf817cb0e9162.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121776.48/warc/5401ca1468a4c2f0a7387bbd70eb0e15670151c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121976.98/warc/70cc7327317f5e831fc7ef5099530d474b4e04fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860121985.80/warc/38d9c4ead892a754f6d2576768b74c7f2972207c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122268.99/warc/f851abd4463d6070c62fdc1cbdc9de455198e9ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122420.60/warc/1581ce81d197476cd87f0ddddda7eb52f4efdb1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122501.26/warc/16a8fe24c9efeae7bf7573a1928d276e8b767f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122501.26/warc/3361814ce982830724643bcef6964284572a5b2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122501.61/warc/980a8a576481baa26047ef029379bfbfc2fff53f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122533.7/warc/59483b373102628da95b6eaa88e291bea52b9de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860122902.86/warc/55c669b877d5c5287b4e67f6a1098733b7de72ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123023.37/warc/134c1d198539a2c911550077f906b40399804944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123023.37/warc/73c23b4cf66861b62564d256b3f9e309855b0e0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123077.97/warc/d6d2027c2f9e2f1d9a469dff16ab07dad0e9003d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123151.14/warc/b8cd303c6d09287c8dce6aec96116f21739146a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123840.94/warc/437a49b5883a1c364d864b697b486e61fea68546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860123845.65/warc/9cdb3921c2108e4d0b7a45ce690425f282010b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860124045.24/warc/7506cb800750918df8a5770ba300b1dbddd28dfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860124045.24/warc/c91fac6b543f122bb0db56be07c16d1aaff6198e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125175.9/warc/afa77243f289725a762189abf4f74cc41a4af8f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125524.70/warc/d334ff11403fbbe3eddc374402fc6fb2fcca2744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125750.3/warc/a0c5eff5dd4020850bccac9d3c20fbdf0d23b0a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125857.44/warc/5a8348ed5fb41db3d34c885fa4c801e15d49e88e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125857.44/warc/c73a75a5b8ee098637dbc4fc6626983e9f7d65ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125897.19/warc/57fe6f325a32e24257ec7d4837603bae03d1615f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860125897.68/warc/3314410df254dfd3aba14ff6874ea61ecbfaeceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860126377.4/warc/bc3ce421f08d9bec835cc8ecd61477bad7045b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860126502.50/warc/19c98497772ad3937316896c8cfca45c8e385feb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127407.71/warc/044ed577652d3e23fbb607c3a75c8fd038b929a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127407.71/warc/0b2df43eaf3abe8f874370a83fb142133ed5af46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127496.74/warc/016c72085c0b6bd2ae9bb254fc36699f9e721b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127870.1/warc/bed9e13d5286a4b8642895ed1a81534edfcbffb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127878.90/warc/9f3029ec1208da918896f0e6a68b487335a8570f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127983.53/warc/8f35adf123b9b76fcf0d55b63b5d14aa3c4c7fb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860127983.53/warc/9e3df30c6d2cd4414e0f3ae224b060b04ae89cf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461860128071.22/warc/dac057967b388cf93f9786ec0f0efcf9b3a172e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861623301.66/warc/938eed11700dbe91d7ebf2082444d34e98f3533d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861700245.92/warc/9b53cc29dba0608d2ae7b34583786d075ec30a40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861700326.64/warc/dec4ef0832350932ad37deb4a37d8b38d3d38106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861718132.40/warc/0c2faa101e14691103f441971b38433009369900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861718132.40/warc/1ae39fffc0615abc582a2ee914fc01dbd689422b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861727712.42/warc/32cc6e6177887a93b3e4a2c05fa66c97a7264e01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861735203.5/warc/018ddac1579ebe3ae11a6f184c67f14dcd4c9fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861743914.17/warc/d1a232101dd9929fc3a7fa08f5f14a070434cb9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861754936.62/warc/5f0e0e8208f15c42337d8bbf1f712f7a1ff71fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861754936.62/warc/6fb43bb8ba0fbadef7374094271177b785674430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861812410.28/warc/c9e586046dfd6bd904052e887c9c35aba9f78056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861831994.45/warc/24e2cb778678ae151f9dff6b5fd1dc537a51df13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461861848830.49/warc/980ac1b141e0e639f619c851dd4d4edd7770d4a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461862047707.47/warc/38ce08b15804b1d074b34974532afbe10a3ea8ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461862134822.89/warc/94144cee1e7ba2459eaa3ba315d1c7937eb9691b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461862134822.89/warc/b9394b6f335842146ba7f6155352182607a77ca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461863352151.52/warc/e0fc32c7721dc740f7350b08adca46b34a3fc9fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461863599979.27/warc/c1ba422294cab9573905404f4a33826c76ae28c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461864121714.34/warc/99db6148ae27328dc31776dcff42768dd31597e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461864953696.93/warc/6faa814cc5c3e5572ef30e3459d4adf277cb54ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1461864953696.93/warc/af4ec0b0bfe1bd0692068f99ba3e6687331ee876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/158ef7bf790e2fb69da621ab6861e78e71727d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1684a93b2057f8bba8c4f9a7afa77bd03a10736b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1aed6a68b7a771d2e5a055c624dcdc8978c6adaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/1cf43afb51ca1b0ec75d938fe0af7533c627bec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/248cb67f09f0b52ea7d4d33741e3b77db497a155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/24fb4897186d14b171f23813a383f0105cf5e943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/26e088325d420874e9f5cccad579d2f1ec362762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/27a51cbdd640c9587b07fed4a9de57f6bfed6e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/2b41fb890c8e6ba96bf39408f2b153ca8b66e8ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/32d6aa91901878c8947700d428b570702fb600d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/331305cfbb923745711244cf51de7a2c34c56d7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/363db31ecd063cddf470dfa8d1ec12af43cd72c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/3f86a1377b7e2ff521a7005441f1a11be623d91e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/401c95008bd6c529e12271f7bc73be684cc58cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/46975be847ffc1100b57b1983f36cd3033117be7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/46c459bd0b35a33be74194af79be63b68abc1623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/49788ae4a27acb20a4e3f174f4772cae2ff95bec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/4cd035f264e48c82fd07f81d74b1c8084df503d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/4dabd59b2719b09a2e8e6b3f09c5b94826e4c9ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/52c5eca5f3f796514d794c9b2e632471a93b828d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/54f084fce48093f8129e93f0536a8b980021a9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/559a67e177a35aade0a6c81876226ca97d185341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/5761a823a80c6f4a74b2f3414a7e997a0169e858.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/5a96f0efdeec604797ba7e6c68657a20c33e2e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/5c96a5e38dffed533d535ba975a2cd73a124c601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/634f0cb4e292fecc19a1d6a42b69974793ebc4a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/6638859e08cb90f28bfdfabbd5e93b9a2a659995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/677433b2012fde798406fca9231cc2fb87d33c98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/6953b6996bf8c6c03ed89f66f7463498513b780e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/6bd8291ff1dc16c4ec5b2b54fefd9c64a665a714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/6c6c7b84a0ca43853c3feac3984850f0c064a90c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/723b42e25dd82b7957e9789608ee084c9f9650b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/742972941e5145e9a7bd51f088cef53d8e7db0f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/74c1582077cc6709f2e9da265688d4b2632ad5bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/757a10b2ad3b93529a5be40548180b1069f030ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/772119f64522da486c16c4375e856a4540fb5f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/7750a85476d65a6e2364cd6870b540e2df3eff97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/777218916527168a8a1e2f4b4b82be0ce85160fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/7826c2133b2239c1c4cfc8849f49e98c10a22470.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/7c1a65251a7e819a3f231f8ed527ceb4cb683bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/7d4410687de96a9e6c09d86a186ccad279bd5e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/82f5a677616437b199f0d2595c2ad22fdcd01a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/8795dbdd059acb09036fbff8fca8d207070a90df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/8837faf405448953f5979444fcac54a4f2d25143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/8bfe06ff3c24e6362f4bfd61209ffad74a33fa02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/8eaf49f79cade5d0144d18c222035088ae2146af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/8ee7cf8ae3360626e87c05ce2980d96a644dd354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/930a7a99293ba87ad0f4c31ff8a7d2a41e0f4773.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/97a9bbc915e3fc45f95947fd84a0e95721ef9d42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/9a7e3bdda14d071d4b88382dbc00843aec121df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a08ac0617e0772eb1ab3ae3ae5a92cea4bdfaf2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a26db384b09066dbd5ee2a6d7eeedc36be46187c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a27d920af1828e04982890d8d2599b385276c2a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a4dd97fabae3d1074bfdbcd4dddeb40a12769116.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a7d2320be7cbcbbe3a9fbd9979a395975e8b152d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/a80e7d7793afeda00304b2637f1b66b26336aa36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/adaaa4b882c66cd7fe42c6eca003ab431fcd1af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/b084ce48a6d3569835afafb7005039b97fd40b41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/b4ed039206b959655fad6eb8001788bc459d4429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/bb034b307b6c089a0bd59e2060ff0922657ad885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/bd95f38954abf249e62fad4d7faf410d4615a594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/bec9c7d09decd070f95ecf26489927aea819c9d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/c12c399afb55671ff0835346aa2701cf275ee683.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/c3531ca35c2be1ce6b5b4b4f205872b257e944ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/c3cf54d0d2437d61b689591932f403747f193480.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/c3f02465de71f054fb5b8b4edc4256bab0aa895a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/caf2082fc32860a944fdf22127fc08b8a8abab66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/cbfd57ea1604e91dc96fbb64ba1f55f71fbb552a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/cd4c4b999fb3e6a1c1188b3ac9a4e103ee4cef8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/cf4b55e8d4cadf89b3dcbd12f553bc8841898bdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/d15ce56fecfd07de5dcc7db3e8d9bfd08217c4ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/d5b1cbfecd89298cca305b851011079f03a8ac62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/d9b29c1ad364973c4ed82a08b6a4b995d42fde23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/e60d6e1f494082bb152f8517249e3327320b5d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/e6a0bd1b57c8cb56e2302918a183c558533d3ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/e87cb946a9038736559cdcd681fcea7e63786137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/ea1a0e2f2f999aaf037b178f46778c85401fce27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/eadc02166de92c881d6627650e9e076821dacae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/eb4e806a3ce92194acaf1ef06b00f84289f21afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/eb986c7541ebf5b8327f0508736a471c7c886634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/ec59e80a2939ff2afdcbaf6da7c3a26039c3707d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/ed9eb6686fb1e08a62ab57a8f01839b575cb4bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/ee6fced157c629891da3e7c31ac9c3afca33092c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/eec8196f4d6d3f122e7635055c04c52a06204040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/f1365ffff6a09b2f4d4f3beda95b40fb341e327e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/f1f1f5a5eb0c3f243648e0e303f419708b256d4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/f8b904d48a65e8870606bf6fe171cb186e8fec99.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-18 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2016-18/mix/input" + output: "/tmp/cccc-CC-MAIN-2016-18/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2016-22.yaml b/configs/cccc/mixer/CC-MAIN-2016-22.yaml new file mode 100644 index 00000000..b0f6987c --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2016-22.yaml @@ -0,0 +1,359 @@ + +streams: + - name: cccc-CC-MAIN-2016-22 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/023ea1f78ccf5a1ea55ebd2a50a3aea042770ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/055d785616aa5dc26a6cae954940482226103bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/05962fcd5c51e669cfe27aa1bdea8341d0d3f4b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/0814fd8f8a2d386dbb2ed77786c1da8217e07ad8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/08726f42446cf23cfc9dd941835e21eba9f8005a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/0b9fe34962908cb8a10e50b84d2d621d1ac67a16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/10f70db88b3361d416b08781eea30d677ee6b7f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/12141e79a4c94a569b730430de9bf6e8065be3b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/142dc6b43957d103a28cd48bef6fce63d4e8dfd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/144f1babd21818c456f3c150591c759d4607a07e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270134.8/warc/0fb5b4a622f2c29c846b8abceea4cefe30c539c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270134.8/warc/3ac42c4c01cf52788b917d6f49cf443e2ed3c083.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270513.22/warc/0efea004653150f3a705b11e207f38b3606152d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270527.3/warc/516aa8952c8feac97c38d745c512ba967617e61c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270527.3/warc/cb3637f6b39ba96c1d6b83b1e27461148869689e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270555.40/warc/9834ad27cd29e9b77b664a0de43388bf54d5ed50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270798.25/warc/0265e4f0be8c189e4387eaf890636b08d763f3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049270798.25/warc/62816630b2367f9396edc1e957aa6b18510bfbde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049272349.32/warc/aa8b012de8866c68de2242bb815f718d391bbf0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049272823.52/warc/8e178fe613a483fdef641f3f1252996af500ca3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049272823.52/warc/9cde0958002c5029e5b527707a209b33c55ddc26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049273643.15/warc/b318d8c230b6330946d4d405d1991685f8e732c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049273667.68/warc/43476102e3a323ff817988bd5218e2c9f82b3db8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049273667.68/warc/e09347c89d45eb8e5311efef8ff3d54ad1236a40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049273946.43/warc/57a2cace31b8d7ed5458d251e9cb61c11c4a38b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274059.91/warc/9fc8181f4b3594648c68dab2bca8ac13a214c602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274119.11/warc/61787afc0288ea7fe5a300c463a4487b0431a877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274119.11/warc/a1ae468b1f6f6b0398f05f56cd798b361e0ffc39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274119.75/warc/d07fabcd0068bfe67392b4d5549326b146332bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274191.57/warc/1d01ab15e3f94f6955c49f0294a8c710ceaced15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274191.57/warc/74aaee7cb23b09dacf86b4319364f15cb4d6a5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274324.89/warc/b6f1914763c6fe4070e589e7d3846a351f31cfc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274756.45/warc/19c6bbfa5c3488ae53481602bc3f2baeb0447c5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274756.45/warc/e35c9720b6e71c8f7bc9896445397161d960725b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274985.2/warc/0269eb121a01abc805577757402fd03bc201c081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274994.48/warc/4c852e44747891dac758d804fcf05f40eac3a10a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049274994.48/warc/99d8966795f76dc634a3b59d2c1ffc9bc2252a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275181.24/warc/e5c577afde9ffb447122dc939b8156c24139a91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275328.0/warc/d9d9fbdab3dd27b54d14501dcf48bfd87fbb942a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275328.0/warc/e3ecb20e231f5c05620e9097066ecb71290b986b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275328.63/warc/9077ca9868d245e0c4df6eb7b5ab6f22fd13407f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275328.63/warc/cd6c34bfaea11164a222b72d55a96de103c5f74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275412.55/warc/a9b19de334198d437ea50402409d3efe7f15d1c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275429.29/warc/d594d52c935755abc35f6271475e97189b158a2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275429.29/warc/e5a8c5a3e23d7dd4bbf3da70eac8b623273df70a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275437.19/warc/39224df12198eacb69e69a7c3476c4851eaf8c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275645.9/warc/14e87e50d7f82532f62ec44e567bb14af84135e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275645.9/warc/7c9fe30058934d6ea65d0f244717e027d41651e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275764.85/warc/8b11f35229fc2e111db197babe4734231e5d1087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275764.90/warc/9c62620a99fa2deff9af2fc427aac3116d66a9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275764.90/warc/ff659def4ab6bdd8fb093def4007d4f3ee537b8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275835.98/warc/fbfdd3b80876a5082ae5f87bf8dfb15b05e00b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275836.20/warc/683ca3d397a77eebc0f04539c88e43451f2b41a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275836.20/warc/8b56a7dc1f8a1114c5cf19918f29f291054c7a2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049275981.56/warc/b7548128fade09e691252e217aa3038996e1ce0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276131.38/warc/78d390fb8905b438e598cd97d973f6a270d47c9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276131.97/warc/4cb47542a588aa1e32cb9377e1e037beb08ce5c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276131.97/warc/c19de4ef92daf6f96c61dcef7365a2183efcabcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276304.88/warc/abe6fdf469d77b83e368951dc82c0f42a1860b7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276305.39/warc/25088c435875d7f54b7eb682d28d38b5e20c9433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276305.39/warc/c4323f50771819323a273b60fcefa2602594aeba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276415.60/warc/e7604ac6da1a7b82d8c0f2e9686add5499667b55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276416.16/warc/60e17c11e88d29b41b3d7d420742f5a6c5597aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276416.16/warc/ccaf084a2f41d7d9f33e75192c958669ee9a91ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276537.37/warc/3cd46bb9efeebd32855287f62d3041d26a6e8b81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276543.81/warc/848a619fb042ab5b52f8cabda18a8e07c7cdbab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276543.81/warc/a88e7510542a296723cff09b8f66175ba14cda97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276564.72/warc/58d6a7520ed6b55afa987ded435882c87581133e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276567.28/warc/2d153999ffa3bb774e599d52100165e2c134fabb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276567.28/warc/2db139912cec240345014ae849855bd5faa7b734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276759.73/warc/b3b06360873d02cba3010c16d1e0510333d1cb34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276759.73/warc/d0f6282efb749f6f69b47aae98d9ab383491200f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276780.5/warc/ba10909ef4f66091c4d1a4ddfce74c4e6885b19b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276964.14/warc/66c251bf3c864e7cfe7b5a48ecf0eea5b79276d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276964.14/warc/f2c5ec9f2edf18aabdb055861fcc17079f08ba26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049276964.77/warc/2ca7ebee667818a2698128b84829a884a3a7f394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277091.27/warc/50a09138b891caafe6a22164c242374521dadaf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277091.27/warc/d46538887279a170eec39ac60d68cc6154631a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277091.36/warc/8a1f5b442df06210d7978ccc8cf03b100fd117d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277286.54/warc/4ef9b1ddfcc7a7e67f42fe2cc18c313a663a13a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277286.54/warc/ae9b527b7653f8d73286a84111be7bb943f515c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277286.69/warc/fae85c602b92638a906aa681571e6bda8f0f35df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277313.35/warc/3a19209da26a06b10f4eedf21d87f8c62fde823a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277313.35/warc/775325fa306173e3667255372553d85c6586e7dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277313.92/warc/80c5fefbd0ef7a5f34ee415e0589c7f0808e9270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277475.33/warc/8ea5f45cd0a7b164c91c817e2f3ce9a08df2060c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277592.42/warc/0c0eda7f745ac7ca96ce41915832f31b337c387f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277592.42/warc/f25d8bc4dfa88bb8c77c50c06dd43652e4fa7791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277592.65/warc/cd16c124fd9bb881021c55ca8717abe3cca54ff7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277807.82/warc/2b590016f1b453a4843bc3debb77636da2171736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049277807.82/warc/40fdad64fdcd11ac4fd9758f89294a3398c7109a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278042.30/warc/92ca1d7d793ba4cbff51bb7207483e8146266038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278042.87/warc/3373322d013efbeb5c6e2e4703f1fd443cb402c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278042.87/warc/895bc5eb7a0641e164f3f1f28b6fa11506dd9916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278091.17/warc/d395e750b0ce00fcb6220b8d6c5f4cb881c8d715.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278244.51/warc/126614cb87083b31ea7336ab42c0d291ecebe895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278244.51/warc/4ffe9a6bd01814470bfb5ede010f8a350792037d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278244.7/warc/9675e7ca2cd9cdfe841725ace9e43a5d57eb8f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278385.1/warc/03b915e88dffec74182889aa59a9c376be4341e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278385.1/warc/a05f9958fd181adb185b510308fb705a1fbdf343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278385.58/warc/428267b8ab6b8b2cdbdce8813d1243b6d8c10877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278385.58/warc/483683ae971352e42ccafa5dc7800bda4bb69676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278389.62/warc/204ce28839dbf849338ffe3756b351cef82b5ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278417.79/warc/347be713079ca852525b610be75817046b64f13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278417.79/warc/fc5a08bd50ecaf7699ade2532678dd80283e6e52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278887.13/warc/f1a07e5148e8ca1afdea9c510c49ffaad644f2f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278887.83/warc/79b08bd4b14d89a9cb800cfdc94e75817f913bd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049278887.83/warc/a5cd9275ca71883d2e314f2dcb5f491f0948b69b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049279525.46/warc/7599bf410985ff74961c028bd2b9a95e235a9c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281363.50/warc/001a484d97f5aec260e785c4baa029b218d8fff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281363.50/warc/3f51f9a3779e7667d738b64438415bfb8a3b54d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281869.96/warc/6f42a03ac598c0d82142c7943c1865e5ddb61f50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281876.4/warc/0298570560c6ff303bbd8bd0244220bdd0c7a7e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281876.4/warc/c7a2eebe43b54eb59499a00f8b254e35440f09ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049281978.84/warc/d9217206be366a59dc1944fc4413b46f596f0106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049282275.31/warc/679680f8e9f49461fb81900161dd23a6b3339a0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049282327.67/warc/180b0e5c1ddaecba1e90671b342397919d2b4700.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049282327.67/warc/c6729985e5a3e926ad773df50b911128cf82d8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049288709.26/warc/3f51e3bd6da449bb6d46a54e11f843be27eda6f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049288709.66/warc/afbb8d9756283db9889c9e3e6c0acc52371af048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464049288709.66/warc/e6c0a4d071efb4d237a7714d56d11a074397654a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464050919950.49/warc/a3632fb56660935f1524ef9883ecb525c41b0983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464050955095.64/warc/8336391db7fd05df99ee9bc9c47001e3d1acae5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464050955095.64/warc/d9d0b414a1f02000e904d3787909bbf4a24abf2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464050960463.61/warc/dd7ad3b4f041291df52495d8a1c31f54eb02e660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051002346.10/warc/0b68a26193c26b3ad41edd4b514738aa33d77693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051002346.10/warc/ee0898f4a0e84521a899ae6319ad942db107462a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051035374.76/warc/9ee7dbd40264a53adbd2416f4e631257a542d3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051036499.80/warc/330655527f813bc6a3e519e5c4d9d05cdcf7542f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051036499.80/warc/c2ea0fe150acfa577643fd098fa5efe707061477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051054181.34/warc/832c63eada2d15433c3519e6c269331494b10905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051054181.34/warc/c803780b30c10c1188154d6179b8ddad4de56305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051113990.53/warc/37e738546884dc98b4bd219a735e75026ebb8d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051114647.95/warc/05b2a5df768e38b4bcef643c8f9d2131e9d48c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051114647.95/warc/d773a47e425637f79a866ebd92143ec26273aa1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051151584.71/warc/fa1d403d588d40c308c0ba0f1b388488cf9ae890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051165777.47/warc/9913c77e6184aade7f5c77e0747922b00b7c5018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051165777.47/warc/e639a5fd14dded69f8949ce088f54c1f6c7a769a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051165777.94/warc/5eede89118cb66959a62025c16a5047a35be3943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051177779.21/warc/4d5f39fa2dff26dcf4c24b175bcc8d279ff74353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051177779.21/warc/d527a95878eea1cbaf93497be70bff1884893836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051196108.86/warc/91ba6ce91790035a29fe3ce0980c9a333f8f4d3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051268601.70/warc/063914b63c4af790aa7ae75b97f6e75b7f512eda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051268601.70/warc/23f91cc5d98b662f3583c36d4b13b77baadc0368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051299749.12/warc/f77126f967bde05e49dd822a12bc9684d3259818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051342447.93/warc/5985af3d2e4d0c77f8018cc8cb8c0bd81314aed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051417337.23/warc/1d11e8f87c7b525c34140d00470f61b1a2c83dd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464051417337.23/warc/6572f69956f66a3acf9ac3bd187850fca06fa89c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464052868853.18/warc/2144eee96b1dab82c2fa1d2d1a6aa1b68123b0b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464052946109.59/warc/78585c4311d326f97aaa5e04487643a940e78df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464052946109.59/warc/dd5cba4c93b996f7bd1f2f3cbccc9cfbc4f051b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464053209501.44/warc/0ba24a7f2e4caadb01001906cb31c61a344a3818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464053252010.41/warc/222806a656b6396ac05e3dcc446763e080bf52ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464053252010.41/warc/6fb19637cfe613f51fc9b98912b2113e81e34b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464053379198.78/warc/97825dae809018aa11326963fa277d43a6ff84e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464054526288.74/warc/b4f3cc7be6d4496aae30f6b591b37fec7074f4ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464054526288.74/warc/cbef754985736177f34bc5119d9818ad5cc3fa10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464054915149.6/warc/09c786d1ac3345c7cd90096c8a9cc61a49efe6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464056639771.99/warc/0d850a5578e7707567ecabac998903f1fe36d692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1464056639771.99/warc/6bf6c192177fb23ace50b3c11d46ca1e50aa7e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/17b0e7f3dd84583a2d2fa0e9a47f5b04a60f4463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1974fcb6d4710a6f35a6f12e03c8444778cf1e93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1eb5205bc7dd9a6efad042d6f01310218285b4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/1f7828ebda49f82c375b12f3442f1ec1f2686807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/211a85f6c0469820aaae9373982fe9333a57b3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2248a0cd43745c82c8a220af7c563d1ee44961fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/22c28bf13d10734a80c9621e729be48341542bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/230e88e80858dd95d62a33e441f49de9442e2b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/233eebe1c775e80369632afbde1ee26e8e848455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2399532c26fb31020a71455b15cb54902e5e4d32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/250d520ab0c3f07ae26d1c6f88a7a3ce735da812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2a375d1b01d403b42a8c390965ff9e221dd27507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2b839d5691a9b6fb1254449d2149753eeb015fb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2bad469ee2f90a648fdff712ee3277010c3c4b34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/2d15f559dc85020b5ce8be9b15fc7590a8ab8390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/358c9bb8275a807201a0908e23833bb4737afb3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/3975c3373cdf0bbd4cb52660309bc322936d8bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/3e9a8efd9c9bb9da333b51639cb233e262c73e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/3f0679545ad58adc51cdc28825a3d35081bf7208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/4996b1525550fac0b2c3c1c2467c05c5c3a428b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/4b59eac56182b54b6ae943e4be3f1a458fe81d3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/4bb19ce01ccfd81beba8382e10d16ffc8a55a58f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/4d35a3b6c67c77d72b6cb6b15727aef2f1d1dadf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/52b536a775eee9d25964c6838f784c09e24252b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/574c143a5e310cea54a7f218fee8dfba1b04a1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/578e65170136d902d61da7b2004f3dac8306cd1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/59da9d1aa9db9f2dd90e6459ef7995fa1ed7d937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/5a0148824222ce92dc83ab7b8d76825bbf67985a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/5acc8f97fac02e2ce29a457a0f810498df262085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/5f6845e8d47325f44cea26ff438222f62722f247.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/6047da21cf9eca3aa5b7ef93fc47bb32da04ab93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/63559705be0ef993f24c6d07a8205c22da9e41df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/64ac8e08592e19f84adea8141897c66115f1fedc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/6975452b748edc7478bf1a832a2f64b0eab5e173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/7045ec3196ce9cb88bc2767df082f38d5a58ece2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/71eed93d0d4e37d469b3a611c2a11b43423c20c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/767a4f981517dfb128b3ba51a8acc09a90f74c72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/78969b4de73f762879bee3843a0a437514ea652c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/78f5b8dc5d35e594898cb40237a54aae6ca5c31b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/7ac552cb4703eb1409585cae597265720594da0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/7b074a8e2e9d1584cc36c5bf99240b0c6cd50c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/855f38050a301f032ed3560f761afadbc622eea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/8992606395345161e1a31d06722122c4380f2cbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/8d936edd8509b82e241ae52e5c312d5326263d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/8d9cd30a584cc547c3cb91d7674e810f4600641b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/9074e2dbdad78ed17872e2f669a714987c854472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/942dc9509756443b307a8804fe5d938f02349a9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/a1fac1598fdd1df4cf51b3a6888ee06b025c225b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/a414efe5be6fe4b54c55be56f7d009ae8ca0d567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/a68c8023116447ba14bef2d5f8b213fdc57c87ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/a7150fe4c3df17f972cd937713529a238c7333b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/ab0d73f704ddf9c31bfd967772b453d6849af231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/ab753698e00f3268b5addd55b01989eea4d62336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/abb644d84fdb8f149723c5729e2e67581feb2e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/ad1a217a432b9bcc588af949049e2bee870b5ad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/af5081cea758e1717e02aa320ce02ed23a995ce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/b055aeb284e0459205068298ac7dbbd5f6423523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/b51502ff0968ead4ed835f3606b9959142893d5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/bb167d052c76041269b835b3bd65c0be85adacc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/bb5113c45defb7190a44c86053fc11e31a3706e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/bce00ad2ff4d12cc07bd40f075a7d577f02da543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/bd7a80ef614ab14d58d1ff39611533c4cd9d5eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/beab53c06450866efe908c98e2a3475f29eea7d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c0345a3d83a8892b2681b7dd5c19ff0bd783f5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c0508f0f5dcf587329e46ea89356999f87e11afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c24317d8365ad034e6a3de9903f8bd1beaf35d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c2c4b3f2293cb989f4768a280a44e384b588b96b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c48700585a8251edf1d201dfa6cadb37606ea53b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c4ca4dc42c561e5a6e4ea10735ac2baeb3d58cd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c60498603b072dc4507248c4e804240cd51a78c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/c81a66558c2579617cf3cbd605f9afcce7933f60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/cc4bdfd86ec3c8d4ea755a6d40ed94399410ffae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/d27280aa5d1b3a138be03279203654c0a36eb584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/d3f6e2c0292f048bf8a994bbfc56eceb076ed243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/d6faf7a0cfed05b2d68f68653c713a1d5f4c4d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/d99dabd2bb788580da602d2dc4c69b1c5cf6f01c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/d9e627212c882b1517e67919842df55077493652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/da381e6a0aca8b5632c207b7e545d450eb2b3108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/db5005757fd109e6b0874ca3fdf450d376b00f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/db6d9621f907e8a34eedd1ad2d986b646fb35ae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/e7beb8a6e19dc097ab5ede4238c665fe77077213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/ec1a4b6adf6eb1bae29a5a357f43259bdc81164c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/f3e40c890ed03e3da3e585165050245603466c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/f4fcf2ec2027064eb4561b3fa77818683302a8be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/fa5edd733162ef33ee24efa1297b4c71ca48c744.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2016-22 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2016-22/mix/input" + output: "/tmp/cccc-CC-MAIN-2016-22/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2017-04.yaml b/configs/cccc/mixer/CC-MAIN-2017-04.yaml new file mode 100644 index 00000000..dbf9b8fb --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2017-04.yaml @@ -0,0 +1,687 @@ + +streams: + - name: cccc-CC-MAIN-2017-04 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/04da946a14c8f871e25dccacb1c94c2742aa8de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/05e7e171627362d64902e43fa1156b28fe06688f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/09b102d0bc9716726971dca1befb221dec351440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/0b63dea6e1cf95df4fca723be240a84cff40faa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/0dbce553bacaf7c6f18b78df783a6e03318222b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/0de1435023ba0f74d00c0c27aba0c4a0241deef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/0ded2acac2484b3e8fee9c0cc1b84badbc8b803c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/0ef536414b4bf3aec4597fc6ded23219d795d323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/103036c86cf46553dabd4f7be1d7a09686ba7604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/12805b964a5dd868149aa697012a9fabbca15046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279169.4/warc/66139f14c3b3f67fa85e19f4a107b06f5d17e0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279169.4/warc/6e105481e714b0ac47ea5c5ccabd607795cd6956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279169.4/warc/aae499c67871569c5b12885be2b1b741e77bae42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279169.4/warc/d4c0f258203168cd79f1f31625021033664adbc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279169.4/warc/e5d7f9693ef2e6faaa8550def86b91fa9db19c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279176.20/warc/39efa8a97160015d55c226d69ae4d8bb7664aa3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279176.20/warc/68a7b952226b3041a9662da6aa9cd83d71b1b8c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279176.20/warc/9caaf0bbe0a938fe6e5a3df30aa370f5df95a74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279176.20/warc/d31b3e5e75094b25e83148786ae8b0fc191fafcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279176.20/warc/eb92f2094062f453521914718a35c15c4e6b9bf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279189.36/warc/200d09f58cdb88505bdc56e765d4a0e603439212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279189.36/warc/5f0c142b75cb3988d7c427d8e44e6b15ddc17207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279189.36/warc/a52a7305571544054986433d59b2c0f404972ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279189.36/warc/c90145d8c1b5f805f267a9579887656e1444f02e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279189.36/warc/dfd0b811d10367a281f3ab9d26a4cfda91fbda93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279224.13/warc/0e10ada819072d2fb35f6a71cc7968f81f8f70a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279224.13/warc/1f5e168aefa7cbef33c2161db4549fe4a3c49b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279224.13/warc/b0cab49aaf28d15b4b01878e753edecf21ae5a23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279224.13/warc/c2caea12cced72db1b4c6cdbfb6b05f660f74482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279224.13/warc/dd152e6209151a6c7fd743a1c874835467649376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279248.16/warc/2a051edde3bee43036382cc2425fe01b18948b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279248.16/warc/4b322fcd47270f00eeb5b61e4950b92cc673363f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279248.16/warc/7aee2e057c0887dfc5271246f3f705ad68210b6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279248.16/warc/c43bc7afe7e4e76ccf2d3a81dbff4a46ef24e49c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279368.44/warc/360bfd73ad50f48bc279145e8db6e740a722a8c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279368.44/warc/42de2ed1428d6f1b446c0b06d775c822fe61288a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279368.44/warc/7b6f1d0727511b77d5bc72502e6190e3a157bb89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279368.44/warc/ba0e03c57b96e6c1a576227ec2c85ba1b0eb5c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279368.44/warc/d37032a1a8dc2c9804c931c4cf283a960f58b2e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279379.41/warc/2a00e132988af77e1aefb751c9ab5987ce5a87bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279379.41/warc/3912b12dc3020bc4e5030737ca606fa765664c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279379.41/warc/4d0c7badf6700115a557e74ef7c2c0dac06d6e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279379.41/warc/70cbc47262f29ebcedf01dbe8dd0fd9e13b653de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279379.41/warc/e86f7162108208be8f51081f79f53b18a8a413e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279410.32/warc/4fde12ee41a6a1477981757426d3393afd1f0e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279410.32/warc/80c38949c3b7175459db73ef0114c3842a7c826d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279410.32/warc/81e3c1855f594b322d2ef6db1f2550c470a21e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279410.32/warc/87eac9910df43c06570959b23013d3c545edfe8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279410.32/warc/d06506f8ff1edc1b226947acc6d986a2f99e2937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279468.17/warc/3ef19f8cd590cbb0cde67eb449d3cdcfc49b5cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279468.17/warc/4e6dffa5d5bbb9c3572c817d2fa3f0fcf280d5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279468.17/warc/683e72f7f714d7e5f5c5a4c7c1d9039edecf1e0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279468.17/warc/6bfad636274c3f4e077d41e41a404037b7043be0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279468.17/warc/c0b34133efafbcf9f876de0c38c713570b91cbb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279489.14/warc/179cd0246e13961a9213ed68c1bef92349984f2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279489.14/warc/4158eef39ab47d140fdcfcacd8394976ccbf1f51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279489.14/warc/7cc01b09a5c3055a74f194e3b956062ba39507f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279489.14/warc/a9c2b14328d5b61afa4e0299844931888de3f1fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279650.31/warc/3f4d4266cc05dfc2db5f997ab8a8220d3b5c6f8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279650.31/warc/5c94ebc5082773cd90be4c7247f9ef0f7d4f67f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279650.31/warc/64b4617660ded78eec7a36787a3a29a44b26e20e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279650.31/warc/6e6c37903df9d6b86db30d049afa3fb981a0a7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279650.31/warc/f49c263dc21a353358135ec56b58be18379e0811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279657.18/warc/92ac5869f6ab9bac9358bbad7554ca4ffc0e480d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279657.18/warc/943ddd2c61802415777c312c202bc3d762a27241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279657.18/warc/94a421ba90699f1730a29aac8c3783fba06ac1b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279657.18/warc/95ca70c1f2398a208c85f2f327771515787aeb45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279657.18/warc/dfd0d62b0ccf569f77654c79555db88ae2a948f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279915.8/warc/3d5a03d045d37f45d73b3c281a59c856d4537f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279915.8/warc/4907945bec46ab7075ec2da0a246b9fbd19e819e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279915.8/warc/4b223c84d7019893a4e951cb76def9c2e38b76fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279915.8/warc/80e85f55fd82bd6935b38761df715f49c45ce2ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279915.8/warc/fc6440870d62694063bdaa5616c1434cec7c4b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279923.28/warc/79dd26ae288ca91b7187bd9387e3b2db9a8e0bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279923.28/warc/8e02df4237bb570a2f4be62db7bbfa7ff40fdd5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279923.28/warc/a7ed342ecff6c413f48fdc8ae5a466b4d65bf048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279923.28/warc/a8dd2d06836e5b133615422da8dfb222f02c5f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279933.49/warc/28c13985f76ad6114e7394c1bf79b790bcb5788d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279933.49/warc/68cd5c6a61033cde9520e0d10002e6879f0b1735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279933.49/warc/a57480639a276d30557edc1dcc989c73955e2069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279933.49/warc/ad8e20d0e8f0a215ff75f451e1f9f2c87d1dc2dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560279933.49/warc/d23a63fcd590cf804dc33a45ee874eb3e593ab18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280065.57/warc/4fbea3db9325bd34dd8dfc86e768c52b8df4a135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280065.57/warc/982aa13ee4c2aac13f6a13365308818cae7b91fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280065.57/warc/9c47bcf3ceeb0d542182bb11ae041d176654e4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280065.57/warc/9d948880d72336a0b05c23c701725e0b1ad4af41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280065.57/warc/eae0cacc82129e0ba6c163e0e9ff4076760a5ed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280086.25/warc/215425c902391d37d1e30d7a67d7f3770597c819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280086.25/warc/50965da5c11544b85b1d942e7b3c6fd9885c9d0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280086.25/warc/cc7f2353d1a3f8133fd849755d55f689e0735594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280086.25/warc/cce38731e78475f1d8dc4a1b57f7c84bf2cf4b95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280086.25/warc/f25f84ce8ba823a52b32ce4f17ad17d366874aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280128.70/warc/a03d17417c828c43cdb1e16096d6678332b24611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280128.70/warc/a376ba76106d590b90ad21bc2019f30087ba0f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280128.70/warc/a9805a72146ea3fb929d557764de9d07c7c6507c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280128.70/warc/c6b083fa19f7dc9af68c6284b3c691eff6945926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280128.70/warc/d1e4fe966f83ea194bf0a27b8822253808ff2e67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280133.2/warc/307342726998880e9086405fa486ced9d76bd435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280133.2/warc/3e6fc2e84340792b0445308da63b10fd06b084f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280133.2/warc/8c24738f2f45aa93563ab423441bc3615b4c7bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280133.2/warc/9302a4cb2255f7c12f412ea257e6b622cff79644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280221.47/warc/15e653219737b1b5879afb1b4e926f54c1b4cb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280221.47/warc/38dddca55e2ec8fbfdb311e949f8de359884ff55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280221.47/warc/bddce95a58762ee93e5b482859edbb5b7689d850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280221.47/warc/c44b1e7320c2a4d0e72ac6e361f4c79c6f755167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280221.47/warc/c685211fa52e745eab5a6d25e6326aa351613b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280239.54/warc/8b20a384d03366badd016c5bb10f6b6691f5eb65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280239.54/warc/b204453bc17f3913f845615cfbd9b2fce9051e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280239.54/warc/c26ef64820d28d5cd4bf967fd263e3927ef210cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280239.54/warc/d3243128179e5297623ba4c1cbbdfe3ea8e2cd16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280239.54/warc/deb9b3f10364c615eaccd9a5c6e594c3d7544f58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280242.65/warc/73191a894d4539393ba95ad5ef5615b69cab8b38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280242.65/warc/bb970a6ac4c7ae2eb2f7ce45635615b771614bc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280242.65/warc/ce79d555f284c1592501491a47de8e0f2d6ab6b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280242.65/warc/cff01a6fd4e0d6fa2addbd7ccfb930de2772ac0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280242.65/warc/ebcb2d007d8c26d1763d9e14a0029af892ff9f33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280266.9/warc/0269f661df266c68c71b64da837a3a59147ed4a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280266.9/warc/06aff40c5113f959cc98e8a26e3129ec9880a2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280266.9/warc/99d6d557ccd4292c807d1a9ed214e74155e87e32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280266.9/warc/c24edbe6845e80297da8ca3e8c077b3bfb3a1b7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280280.6/warc/8493207ca4b0c641cb97c5a0be07988beed6f414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280280.6/warc/89d3f6462ea2a25198a2daf907a946dcec2fbb8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280280.6/warc/c2351d94fde428ccb3a8a3e8801fced29457ded8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280280.6/warc/fb009771340d568ba39725ea8e5e6346f18b3fb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280280.6/warc/fbd9d4cf530a30915f5f7992bd436d0c2bfcaf51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280292.50/warc/0145ce53c59cb8d06da91767cfdc4729148e121e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280292.50/warc/0fc9de1fcfb677252b2cad8d885ac7086f6f7f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280292.50/warc/364d115bc01a74c3a21422aae304baa5ad234c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280292.50/warc/49fe6f531fddb82a40145c15c2b0cfb415c5bd98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280292.50/warc/7511bcded3ade9dd273ac38e2405e2549da5fcbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280308.24/warc/35805d6f670c788d444fd94372f287ffd867cd14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280308.24/warc/5af4ab9ab263c44ae49456a52450ce56db7dd376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280308.24/warc/6e58024d19c664543152fc0bec8ad44aca593216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280308.24/warc/7713c38f887a934abe94a1071a91760efc5830e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280308.24/warc/a87e7be97dd7dec97d21e80f1c4a8d46eba30db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280310.48/warc/7b89b4a54bd1889de8bef4b56d3f096d233f487c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280310.48/warc/81b4e69ff00ff1a986e73d21c1cc36560d3bb424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280310.48/warc/917160276326692a075d395759d012d882ce8c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280310.48/warc/dccabaa7f9cbed041a81a6e50b23ff1e63885251.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280310.48/warc/f2f6f57eea36142d4f7225906699af8cc2f9776c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280319.10/warc/51f5da723bc4db2cfcdf460571696d45175b9421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280319.10/warc/676278a6587628bbb67467558150042d41608210.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280319.10/warc/7d2aee512b0991d67827630604bf45729c0e4374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280319.10/warc/9511df54496e80004d712ab129f4a5b56aecd083.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280364.67/warc/08889d54b7caa5dcbecab0533881688bfc629f0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280364.67/warc/544fcdcbedfc581b98a56ec5ba8a4be309537da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280364.67/warc/5938f29a0132878f0f56fab6b5c3307fc82016d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280364.67/warc/afcb7fdd788a515de6375963d55a49151e8c6e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280364.67/warc/c186fff489bd3bb7e1a31c77f42c00a16507aa54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280410.21/warc/245460d32870519ed80513ba319fc300e910eed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280410.21/warc/2a638cbb50cb3c56599465023ae92df6672ce46b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280410.21/warc/300bc013a46b48363f085ab3eb481c8b257153fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280410.21/warc/b1bc782af44e3ee1e681dd1420c2b8df6e913c10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280410.21/warc/bd2c6787f7848d426bf241fa9a9c45d06119f2da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280425.43/warc/1afd75ba02ce5a855b94d7502ae480cf09028bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280425.43/warc/1c6869a4666ef5933673d7938f166795319b18cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280425.43/warc/281e9014d3f333357a6aa2603957a4aa5c1d939e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280425.43/warc/7e16cf68ca6fcbe9c1f17f9332ba44eac9b76f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280425.43/warc/ab7b788f6d6c0e786c8e638ae92b8c5d86b0cd58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280483.83/warc/9d3938b73376e96f7e5fb2fb4af5c11c218ab882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280483.83/warc/a6df55e2463412eb0a56e2b328040879a3eacb32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280483.83/warc/f1617e4cffb2a11df16a797fdd35ceb151372908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280483.83/warc/f34cbfbe097dbc712d5309603501fd69a5240b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280485.79/warc/042dca2541a934da4f3961a47d940fbb7b57ecae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280485.79/warc/d3e47cf29544c44dcd4bb8b0ad2fedaa5ee32b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280485.79/warc/d4eb0cf88e88650f6fe7dcddeadbb29e89ac80a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280485.79/warc/f5b0b95d91d6202783156fc1287d23b221d11166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280485.79/warc/fd96e55ad77c23eab1f4b4e096c461872ef444fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280504.74/warc/67d31e335f958de998df114125b91d968cad52f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280504.74/warc/87d06347206f1d548b75cda96cca10afa1236324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280504.74/warc/c225720624730af48e4164e47bc1d17b802061df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280504.74/warc/d94b1d0ff9746b50e5dc20a3f9a30cc6c59e3185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280504.74/warc/f33e9272e957b78fcc9432ef163f9cd28d538c1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280587.1/warc/6993bb0b5cfab18137a8846364c75c13351c15c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280587.1/warc/947741e36e87343a8361829d65f6d93224a4b4ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280587.1/warc/adb1b5392fd47b183cc6b7f2e77f89cad8202b7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280587.1/warc/bf51046eb9fa4069ae0ebb2baa1d106d7524b924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280587.1/warc/f46d9f90825e119313cfdd6b562f15e2f83d9d6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280668.34/warc/4b2805a8f592b288d033f6c9c6913f4acf44ccfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280668.34/warc/8a331eea3ea300497ec019e752c3fd589e1adc01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280668.34/warc/94379645d1c00db95d4a310862cc2250e42e2668.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280668.34/warc/ae43bc765802f3761233e50ed46f58a4d5f553de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280668.34/warc/bb75d4c16d575d9a6f5d168bb1dc01c87e1b9e5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280718.7/warc/021ae30016b55d8bedbb1bfa0cf5a36f597c9da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280718.7/warc/050e7a9c2b932673fa392ee26f7f885b0fead201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280718.7/warc/92825d70ea82e62acee731ccdde748ac8052b453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280718.7/warc/de95cecd7fdd705ff0848699d16f93a4e4727dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280723.5/warc/392daf9cefac49988f0750d938888e6911845055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280723.5/warc/667ceedce9aaa44fa3b42edeedad28c6c12c936b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280723.5/warc/6eca01cbf2acf3cf8ff40eb591d7ce66413d72f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280723.5/warc/7ebff0a557cf31e4406b664185a4df1da843b84f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280723.5/warc/a8e3171e8aed4ff5cd904366490ba062046a9cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280730.27/warc/317fd4e83e6dc4fccd38be3d2a3f7bf5f3716098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280730.27/warc/423f92cc582034c0de0b41ea2378a623c00ffbee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280730.27/warc/56c19e748d69d730b1a62e018dabc4aa425068d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280730.27/warc/93508c6d00d4be269d9647c7e8101bfc020e259c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280730.27/warc/e6fa391d4cd32e8ec52a14eff14c85dc06065dbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280746.40/warc/1718f178774a5d1aafb17b0d6d432fd8145b5756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280746.40/warc/762012c465b8e3ee1199e1d7c44e4b9cbed6dade.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280746.40/warc/7e3114135693fd8598007e8dd25e0a63c6d6b122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280746.40/warc/90bce885e7d4b3a7b35298f0741c0cc5ce17fd47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280746.40/warc/a12e6d811deefbe7c3f290d163558541470a18e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280761.39/warc/90980ce7974390d3f9a9c43dbc2d484880026662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280761.39/warc/979d08f15a8155fcf8bb937026641b91366f6560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280761.39/warc/bc9629852441ac1deb6cb608ffd8e095b76f402f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280761.39/warc/d8b05fe2f392603a796893dc1bcfaa09e81d19c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280763.38/warc/303e9c8b599f563cf64fa8d4b4b7c48016223c63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280763.38/warc/3a42f5dde75ab2d798e8226c5535739a5eb039b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280763.38/warc/5eeac84417de81b709d4a8a6bf9ee18bd9db1dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280763.38/warc/a1ccad1fdeaf9db78b204963ac148b1e5a1ca155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280763.38/warc/f2de7d604b1eae174fac26b13a279d0b7edc2256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280774.51/warc/17bfea445bc88bdf9e94d863d2086733122dffce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280774.51/warc/3e313f73f7eda2c58658fac78020765cc2388b0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280774.51/warc/46a139717663a2e9e51def9c2b9a8663b7a91665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280774.51/warc/754e72cf3251c0b3ad8d93444b78d455937500fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280774.51/warc/d54611441996893f3f6cca86bf0bea111f71563c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280791.35/warc/1cade1540ff28ab2cae9b08652cf6bb79ac67ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280791.35/warc/497d4b9bd68c17e59674990b3c1644bf2ba49d54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280791.35/warc/965492074a4244d5894a224c89f20818e508078a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280791.35/warc/b8d633c078d771fcb684eb441d643f9e37f0c2e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280791.35/warc/cd1da314fd53b5c46831fc34ae698df34c356622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280801.0/warc/0cba29918fe9d4b9327ad2f5d7ae6061cc1e7486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280801.0/warc/353794529fbeedaa7849e1f7efe383920854aa4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280801.0/warc/425c24286cb4076d17d240ba2d0b43bc4c3eeaa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280801.0/warc/432ddcb86d4337cd7ec47232dfe30847e37bc826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280801.0/warc/5724702fece17b44bcbaefd3aa90a8ff32533b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280825.87/warc/996e948b944533c8a574cdd33099ed7295cf0523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280825.87/warc/a9d957c85315553cbb1b13e7fa8d5bc7487d6f6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280825.87/warc/b458b4095e241f0b6d28ecdb6746c46d0c098e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280825.87/warc/ea18b92421e46c546f9aeb6b3d132e9dcb6bf8bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280834.29/warc/24442a98804c8005ec6d04e2281a463fdc54adf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280834.29/warc/25feb4c84cc5c77bdaae7af1de31136ef855c0ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280834.29/warc/499a68d42d9343e9f8e0204709adb1c8f193e4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280834.29/warc/897efd904f3d9cbe4be2a697c6287467442df9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280834.29/warc/9e6e9931beab1797193b03c8e2e15445d94fa040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.22/warc/4b46620afcc62137d4bebfd5b7b665dd10a31324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.22/warc/b4babf53f72dbc342837b0fc0d37b2bf72165a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.22/warc/b567b184981aa4a26b6ba9773ce88de5a9cd66b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.22/warc/e6556b72dff5aef402ca896fb5833b096508cce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.22/warc/f531de13def156f76f1c7de45e29ac5017f6e756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.60/warc/17aef46fca78618e160deda8263a054056d1b472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.60/warc/4e52bd6baff82b6ce523c1ae16300908578a74ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.60/warc/4f6a5c3676219aa83768f8504c60740eef1abbf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.60/warc/b7e55e34943def1710a27bc80f72660091b23b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280835.60/warc/ffde63cbf9bc1f40fec9175224470f24e496064a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280850.30/warc/0aa306e27734c3b51b55152c081125e198b4ef56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280850.30/warc/0f894f9fbcf16bc738cd5f4c9160b7ff21b83702.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280850.30/warc/278153959b9a3665c13b98f0094aab298e63c4b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280850.30/warc/9b0f5e426716247bba7b720a92b8ca0b4a41f6db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280850.30/warc/c40440b08b24a50a815e4ef65c3e9a0f3186f05c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280872.69/warc/03bcdc8e43f39c6a4a43f611747d64016fdb744e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280872.69/warc/0cb64626de918c4c2dacdcbcbab1d9251d0fcd22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280872.69/warc/8beb3c2b111a35018f4284f4b61644cb641cc893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280872.69/warc/8dbe99ed783bb12ad5f9d5243388aba615e4c4f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280872.69/warc/9a76b14fd933ce9dc8d97232785673f56baa5df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280888.62/warc/012d37439bee31813289928715c365365e8dc700.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280888.62/warc/453b70837c1c4a59bc605b9f35fa84a087dcae34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280888.62/warc/53bf73a9e74ccd4cbbf51ee4701649e2eea02ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280888.62/warc/767a7ed15fd0963249188f478b2794972f3c9a85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280888.62/warc/83bf7097f0c971dcb60ffd6da4fcfbd67f08f524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280891.90/warc/5a2edd90332e34d312d35ac73ff88377cfb9551e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280891.90/warc/61cecd11d1f5f27a9e8ebb69f437e256e1b28baf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280891.90/warc/7c8ee7cd892538b8e70e3806ccc0142834c899f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280891.90/warc/80e13c67cf75022fd380552d25093ef056b8561c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280891.90/warc/f05867cc9da5a19a02a706d8363cf9c66b4bdcae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280899.42/warc/50cfb8aeb8cad8b303cbcd642b25621339fd944f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280899.42/warc/64f0251d15100ec282b8210c05a330c0cfed4e1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280899.42/warc/8c26daefec46a171caf13a58227b11533619727f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280899.42/warc/a366115141b7f70dd595921f25ba3f9d2f9c7f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280899.42/warc/eb58164bb454b76cd725a4ce5bd0c92507c0f4e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280900.71/warc/8245fccb1009e693c1349ff8f8b098a6bfd940dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280900.71/warc/a3c162e74ff1117af422e9cb8d2b6f582829f894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280900.71/warc/c97a2e214cf9261d257510ae0fba8d5d9ad2adba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280900.71/warc/d0b21fa6596a7d33fc01d9e61e8e8ddc99363ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280929.91/warc/38e850d20b19c396ee406583560e4e8d9de30eed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280929.91/warc/47a5fc5eaed0154c3cb20f00cb35ce882cd8169d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280929.91/warc/a19b4d9d4eeb4e14d2c3831b8962ba6b83b202b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280929.91/warc/b9c81762b176b42b27f1dc6c3e72f00de980a312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560280929.91/warc/f2ce4a1d0fa728199173b884a082b7abc1dba198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281001.53/warc/08d5e435bc3bb26fb2cd6da7d87607fc743c2243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281001.53/warc/24ef64a986683769b3226d0e656ab7546da9ac36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281001.53/warc/398e63b22c5c319624f4640ed5558f12d4021cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281001.53/warc/4f753ee58b35171955dd119069449240398527b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281001.53/warc/56fe6e93da46bd8d2f418a307c9673a099f938ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281069.89/warc/12bee887a2b5657ad8e6b8e49d3f747873136d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281069.89/warc/5524f5081782c338784c438fee026c5a04d988c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281069.89/warc/7d343003619b4465fd5c33e697330ba3e4dd87b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281069.89/warc/9e94fa81c5bc0561f1f5882f862eb5f93eb55c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281069.89/warc/f69b1536e3575c7447c4a3da019627499e99b060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281084.84/warc/501ba4e9f2bf0289ae5b7afacf0c02f145d8bd67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281084.84/warc/7e68844946f577c32051a92f381c7604facdc7f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281084.84/warc/80bd48d0d83bd5255dbc262a71c21e61be48ce2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281084.84/warc/c492d786369ae7a47c8549ff6830ed0ad483e87d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281084.84/warc/e6217e397e482cb86810caf208a2d1d5d5f1d508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281151.11/warc/3f4991aca3180a53c7aed0f9089e037f10f9378c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281151.11/warc/6c42745f026c9536d37af08e77b9492979acf907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281151.11/warc/cce1bf9ec307d6a92dee07135ad2c79a5044f7d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281151.11/warc/ee92c3c9da3892f6fa1bec2f7302b8b50f4a542f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281162.88/warc/1fd0953d5dd9e0c8027366c49612b1af13da7b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281162.88/warc/33663c08546ff6428fc89dfd019f56e05a5c1442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281162.88/warc/7addab39a7181cfd52d147287c318c242105f508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281162.88/warc/d13f0e49b1ab5747bd1f9f12e88702d23794b548.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281162.88/warc/f283bcba69b8531625a9c776d456ce0d4b7cc9a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281202.94/warc/1c68acf55375e1a8e5a16a2007f7f7c561e7a068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281202.94/warc/1f6bb5fbcb5a5486d548d82d45d803f6b4108ed8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281202.94/warc/55c938b811eb76c5b6e242e97003bea570646a71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281202.94/warc/a328276c33942830b8c9ca729d32e3e78333c3af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281202.94/warc/dde038a54a3c32770e21ea6cd759e9d814d2bd72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281226.52/warc/0b7e2bc5bd1ea788b9c4de7d172f3f75189a108a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281226.52/warc/5414bd174013096ea30521d9557554d95bbbc226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281226.52/warc/78a583a5aff9c68c87347efd51f4cb39898798a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281226.52/warc/a865667723bf7209cf86c572200999639d56486d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281226.52/warc/e9372ca45a4f370b773a32095a61ad841e288afd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281263.12/warc/114a153228e7c7865da40e5e129a73727555f5fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281263.12/warc/b42950ecd056577052438d47689693a14c55eb10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281263.12/warc/e4e12157cc70bf357e02bf747b91a14caf6be6aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281263.12/warc/f09afdf5766da4182b3ba97d9eeb4add0b4c2eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281331.15/warc/3e2dbc468d8d72530f653a8506db8a2bab462b46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281331.15/warc/69f1ccd43fdfb9a12d498ec2bedf54f436d8f830.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281331.15/warc/89d56dd4f1b22cc28ade76ebebc90cc6c1910bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281331.15/warc/c0b7a556978aeae9b1fcce3eebf0d7eb25a4edfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281331.15/warc/f551b08fee295b6b8e2b06f4e9617f9a84c7dddb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281332.92/warc/0bea6a97f223ec4e06807d7d280fe46d56d781d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281332.92/warc/4839bb9aabaf9f168b3a0bf65f77ecfce7a2ef44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281332.92/warc/6b0c32283a49eff762856c7d5150003da2e6ac68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281332.92/warc/760bc5809fe98c908f2f3d1221512b8ef79b8c21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281332.92/warc/d7bab5c4ca61480240602273c442c289de4a2f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281353.56/warc/02e39fc73ccdebe263d1530e5151df83060b6935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281353.56/warc/644d26a07df96d5b62c075bb9c73dec27c8c9e12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281353.56/warc/85c573b44161f3322686440749315d9f9510dde0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281353.56/warc/e6c0a28d7b748d67cb32752c4f66cca5d4cd9848.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281353.56/warc/f0519e1a9f9eba6d4281e483dc322151b910a238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281419.3/warc/5e5d50a46e83ef53b5dca09c178f41e816acb1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281419.3/warc/7918ae8249c5023158ca6d683791113bd23a28ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281419.3/warc/b9bfa73d20afe37e321da309dd4323441be2b98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281419.3/warc/d966e456120f6d18a7c0b7d33c2b76fe7af362c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281419.3/warc/fcc2d62510fd023c588dc2903bc390dd29073516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281421.33/warc/328ebeb2c6a82b451fa95e86fb7e120ffba14784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281421.33/warc/39d0519fc6f4f53c3bbc2a0184c036c8ce56b9b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281421.33/warc/b6414db87ceac8a88eb8fecf8330d95e342830fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281421.33/warc/ba13d734d0ff26b2eb2e94c539b39c60a20fc9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281424.85/warc/0a6450e4d1b5628cf73aadffb9dc69a767b5fdc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281424.85/warc/50b8fe744181ee6b9107a23aace636a69c3bab66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281424.85/warc/e179349a18fae62146b35d4aa46493f79ea2a4d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281424.85/warc/ede269823d9f831877e2c8a157a7d4333534a6eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281424.85/warc/f7e1c73df8570aab055c5f066004ed2199df30e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281426.63/warc/7d67a0b557d86585dd3c23d14df322bb9b82e603.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281426.63/warc/99120a0aa1f562b10b70bea6efac490e1c7e7028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281426.63/warc/ea6b454d498dc982e952a8f5f28df3f76836bed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281426.63/warc/f7f3028c4c20920e4eb4736187d98c430c15df29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281426.63/warc/ff829c37b4491d9e4bb2bb07b7c81563c19af4d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281450.93/warc/1bb5329eed56cfc81900422ee52506f1b29b2418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281450.93/warc/1e3f41a2a30364dcd44a0a8006e9c3d6005400e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281450.93/warc/300c5cc52300c485384d39313c38d3c59a88f388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281450.93/warc/361a65dc4bdfa0c565535ae00c754be311d7598e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281450.93/warc/72b9319940f70296e1c6fc9693a0972c79e11878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281492.97/warc/465988933920a739f533ddabf2b7ad2a2077eff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281492.97/warc/5aaacac97972a21f6a3d406073ed7354c027f8f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281492.97/warc/9fcaf78b18470af043a21335dd677258589bb299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281492.97/warc/c46a335d539434904bed06894a99c942440d8653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281574.78/warc/212bbfce22099ce208abce0c2965cb2e0d99b543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281574.78/warc/559df0e0d18c087f1759a9388010e15b79aa21e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281574.78/warc/8ea9e6395f031cf9581559cf5b5188323d7b7cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281574.78/warc/ccdeefaa7a2c901de7e2772f99cee70186841f83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281574.78/warc/f9bbf46ee0dfa07fe7aeb3c2b370a50673ecf089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281649.59/warc/0f578403891db7f3966ba85a854d2694bf6e6869.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281649.59/warc/4d9f6ba5d47659d83f440734a0fa5d2b8c84d5c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281649.59/warc/56631e604350736b57c0eaeb5c37e8bad0ba167f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281649.59/warc/7c87dccf15973d82f7e1a0187171145f0cb73b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281649.59/warc/be4d84b6a920e56f110f22a669d17c1d387af69a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281659.81/warc/7be483a7140c5cf116090a3638c55198ffc466da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281659.81/warc/9b03edcaa18a93a9f626f9f2bcb372ce7b793ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281659.81/warc/b03fab7e605d7934d734554a244931dc01ea4ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281659.81/warc/b36c12d447e96d727c88c32760c7e6b2b771575d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281659.81/warc/f2540ba62d628d98fb9f03bc45fbf213a75ec898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281746.82/warc/615261155a36fe89b5729ca9cf860d7311ee5980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281746.82/warc/90235f1ddb3c52abb6a02782a28fcc5b6fa4a1e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281746.82/warc/a121a81d40811e5122490cc5854ba84d089b7637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281746.82/warc/d26564554c4b503355ec00e29d13b327e9ad8703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560281746.82/warc/eb835b136b3d133fd9afa2cb41f5b8b869e55030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282110.46/warc/32d8c4611f02c70acb0f5fea169b94bb2fa86df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282110.46/warc/4994ddf746124c25fc42512f2f09bb3076213149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282110.46/warc/591c69b8c15d9214f5fb91d3e8824d6e98284245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282110.46/warc/66cf5f4f8a99bbfb50afe722c0651fc53817c71a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282140.72/warc/19e3202be9f87dcf9ab6509e0a2d26d57e859ce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282140.72/warc/278e6171b8256c852108366b91fd4604733915ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282140.72/warc/5f65d3b7d2de8518c5629a1537d253c5c5db6241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282140.72/warc/81fd24864861c1a7e9c73627722592cc856ffbb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282140.72/warc/f8b8c2ba6bc653904d60a1bb705ba92f969e6ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282202.61/warc/1a28f58953ec6e077cfaff4bf3bee24fa66996af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282202.61/warc/29cc4413c8f58e556db6b34c2ea1566e6985c384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282202.61/warc/92d16263c0a06d4a5343e347430d108a152e3c3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282202.61/warc/dc147e5a1b1cafee80fa458edcc2a3ca8d483c8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282202.61/warc/f672f4e93b42b98a6a3df155e4090783ec4bad82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282631.80/warc/0a6242b07455b80b335fd5b165adb1893eef6b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282631.80/warc/0e66887a012377bd610417a8862957437e0656e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282631.80/warc/14fadea9f8abb0335ac0e396589f210ec06e1c54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282631.80/warc/294284ae507b375467161ee37815c36a7588b9c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282631.80/warc/7ceef4c164ca50c0becf7b6210f0432f4970c24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282926.64/warc/5e390ccc3c472f0ee433358d5f5d80e155942f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282926.64/warc/7242c2ae030e64c05d1b769f34cc88c494af8ced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282926.64/warc/9d5bd4ca47c205ffcc6b13121d048b271d022b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282926.64/warc/dc5d014badcd833d3a28ebaedec098c39c1ca90e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282932.75/warc/7d3e1f6ac46149b3c1a35c49865829876e3ce359.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282932.75/warc/7f82260ff369bb1189e7b7b6d2159f91c4e51445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282932.75/warc/9a028af944f477ad904857f87ec002384b976a4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282932.75/warc/c5abd4357c9e1a63ebeb3f4e8a69535d78ac0192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282932.75/warc/f0836553ecc762a99830b9c497a1abcb3c82e823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282935.68/warc/0fe66ce3851f214f7b9dbd161ad8fbdf019f6a89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282935.68/warc/646b1106eaf6a05e2d213ccecfdc323a1c658d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282935.68/warc/8cac13420c3c30f714f17c976d10e903dcb30749.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282935.68/warc/c28c24fd525d914662da6b5f5b9d68ca5699acd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282935.68/warc/fed116054b23f9dcf351a1c35e227c41e1f70b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282937.55/warc/0a4c92172e81b4653788ec1fc27b1aef6905e0f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282937.55/warc/37a6968faaa268108b73854a4454064b249c5ad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282937.55/warc/7f13382293c497ae5772e595d34a26c24bbfa4cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282937.55/warc/bca8e79c0facb78bdd7422a0af9913c7e1f88ef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560282937.55/warc/c168881227c7d41cceaa37017ceb511a7d1f6945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283008.19/warc/0d3b162cbee3941fd79d413a6f61790701d0f95f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283008.19/warc/178437a8d7d54a8f663ae71a088e7bb19edc7101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283008.19/warc/1c9780fa96be68971585cda3fd727e839c8dcda6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283008.19/warc/338903950ec4e9e3914509602c6a4ac35a398644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283008.19/warc/399135ef2b76e5eb2ba245327104563c9f9c8a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283301.73/warc/0f40463d0d8914a2d55584f10927254844408315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283301.73/warc/7cf37ec5b7171b6abb6422752adc7740c623b68e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283301.73/warc/806cea2efe35ca1da70aabdada7cd679f476b28e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283301.73/warc/960d1e17433aa4d2ace74f616a64291b55388be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283475.86/warc/1f8b0f590b47cc98fedfe0f699fdd659bd74b825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283475.86/warc/763daeb241f2beeb094fbff33964c62cb310a484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283475.86/warc/770553b7bbe1894627c51e2f0670917ea92da6b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283475.86/warc/db5bfedf1fe2e0894d1f643e7549afad1d902e39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283475.86/warc/e437c77390be72664240468cd5dae9f6144aed7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283689.98/warc/5171dbb5fa26f9ad3608238fb25bf32c8636de7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283689.98/warc/abc49761ebff5c79012b3724805ea4e07afd2450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283689.98/warc/ba7914a75ab8c96798b9415afa806cb4b8493958.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283689.98/warc/c3c683d3afd1dbe8259984a48170c2c874ff879b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560283689.98/warc/e4934dc5aacda3e9b64d957b40fcfa1740830eb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284270.95/warc/84d6389d777d5e62e21f0957b4a7e7535ea8b748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284270.95/warc/ab77d08ded0f088521ac276a676cf8c4033470ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284270.95/warc/b2fb2455485c1e85ecd8038f306104fd657926af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284270.95/warc/bb31d818bd2d2a00ad1069d965a8bba951b523b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284270.95/warc/d369681b6caec41efc97e3eb94c5b91435deae44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284352.26/warc/0e3f5ecf7c37ece157dd29491b097b939348639e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284352.26/warc/33e640cb2e99405f695ce34ac070696d9dbbab08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284352.26/warc/529d64e4c989ec6073498cdc5dfb89865cd9c042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284352.26/warc/d829509f3ceb81cc4554afb0649f7dbb868b8fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284376.37/warc/0a925510d99d5a8bb87ebb856951405f4f596c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284376.37/warc/27693bfd362503e9b0c6c238715dd7b131af71f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284376.37/warc/852ed626709285bd4a8377c52ff94dfb0b25f60d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284376.37/warc/a079350aaa9dab75160b88797f11ee09d77cdd58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284376.37/warc/ebba0317c68960861a729f874aa959a39119ee8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284405.58/warc/32676466218c6ccc477afae0c8271945c3b99c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284405.58/warc/8b9f1238dcd2507a5489e08cc172c6ac76460c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284405.58/warc/b97513654d0d62174456ef29f094bdd8d9f21b06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284405.58/warc/c7126f31ce2ca9a0b47a189cc80dae4f7a509c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284405.58/warc/d3897506e6da5da86b4456542fe4021135e26407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284411.66/warc/7c2aa9f02684463b80dd5aa944e34a0c8bf96a09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284411.66/warc/8b58a29158df9eda8c1462511a11a7da6977c9c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284411.66/warc/e5212c19d7cd27bbc247e7f44c7fc4559c393d56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284411.66/warc/ebeb8cef00164c92195fee2eaf02ba44e4ec9533.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284411.66/warc/f2aef9cad40408a23e9f0f242525e1ca44f9fca7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284429.99/warc/1961b0d0f8269addee9f0ebe1d99a7838fe1b37b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284429.99/warc/6ad7fae732d11ad9a3c83f3397a223a35237771f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284429.99/warc/91ab78ba6c65c104fc313dc2029dc6239bfef693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284429.99/warc/f17dc9a0ed40b684d798351d98d1582c3ecec789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560284429.99/warc/f63e194c732690ffa8e89f6269ddbde22bbb4f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285001.96/warc/1157c6970cbeac69515e73f60482b1fb55c2583a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285001.96/warc/14b6d0ea7f2940110a70ac55e082fc839325916a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285001.96/warc/bb6ff9c45212bd1256596349ebd50571cc6fe997.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285001.96/warc/f2bdcd0594b7936390f44b3868ef109e85a37cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285244.23/warc/3709e2ed022966d2d47e635d0a4c76f2b084a8b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285244.23/warc/9d8afbec0c262744bcc39ed4f0711663c3285b8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285244.23/warc/a0a7d59a1198ea5ee32df4c0eb9b60751a830442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285244.23/warc/d319c34cb71206bc9b40e293d248926b154ca628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285244.23/warc/d764d70bf995ac15089796f18e542c6b143cbe06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285289.45/warc/30ea5b78ce5fb5fd3e9a0de5807e935e37580895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285289.45/warc/3a60ab75f9637b1c6d55e1405783fe893ba9a76a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285289.45/warc/472c6d081752445a56fabe9651bafa9a4d363a0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285289.45/warc/8d9771914f995daa7efff1405c0e5edcc94f7b04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285289.45/warc/eff2a0e42c91bc9b4185f9b0601c4a58a85eb611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285315.77/warc/149be17464508526e5a95660393b4b81a8db396e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285315.77/warc/26254cea0ee426babaf63ee7f6e107421f8e5d44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285315.77/warc/292fa5319fcb9903e9e4355f2ad987b5578a6d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285315.77/warc/b92b3cb5111e283d6f0f4bc9a8674413a5c22485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285315.77/warc/bbcf5da692d77e24ea4142e841f8a96d77209468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285337.76/warc/2e7dd4a5980232cd3d9c21e0a57510c65406b509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285337.76/warc/368cff2be1b5a769499c6a3375684986d3533e19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285337.76/warc/48b84471ab9307b2e3f4838e9879a5bf9d564539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285337.76/warc/720a991e0164e01332e45241832201b382e28019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/1484560285337.76/warc/79f24e346dec1f4eead1c7061ab256aa49e90ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/25288cf33a406e90e3d16c37a55bee22f76a5f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/25639609241e97fe15270cbda9bc6164f0db1b40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/25d76fb071bed1829dd493d3e1e01abfd052a7bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/2ba8f05aee1debdef1bd99315e8fdc9e19251708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/2bae63c2e39de9f254850acca27df4b5d594bc0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/2e583713d6dfc082095f6007b966400cb0bb6afa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/2ebe317a696313e74e05c10c6e5d1f4869794322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/2f15cd957768782593397c969ef80e344768bb08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/33d68d9c9265cf66e0a9d4574a9d539d6d4b27a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/349588c2ca38ec129688d431cd003c2043c21a1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/379ef1b2cec9ca3ba3a87313958957805db74119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/392137954e691a6680614d11a56c114dc3c7d23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/3cf0b971cfe8d75366443c2b7b9ebfdc833a582c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/3ef5a7cc5dc55f6fce7c91911acc0187af35af96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/4542cbc063e22a63ea7002a13454e809ed5925fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/454c2270904809f3035234026aef2c23ad5417d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/4ca01937a41c9ea62c18b571d85605b4d1c09401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/4dab194dbfec71f84b754ed7d4aab9894b457277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/4df8be6ed39c283234dec15501b7724af805dda0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/4efc760e705fafe7c36eeb7d8c0e79e779560014.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/5060b430e549bdeb630d7a2befae06ea117091f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/5902e66a1de7a24bcac20032f151d57722d5956f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/5a9d73340cd8badd983623508381b30bd05dd26f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/5ff607cb17e0fec95b858a4292a31ee778a48909.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/6173b0e5302d3cc43638022d912b0e6add83ed3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/61a45dd230851c9a674e9f55ebc62b79fcab6f31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/63ecd6e5769c3cb4c5a1a4b1edc10ca56b7f55e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/69eea5d0bfe735061d5512ddbe64b78e5b884455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/6b79927d26b7322a42623405be0845e3643456c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/6c4cb7d60d7af0118c844cfbbfd4054a2c800e29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/70665cb7e4ab715e59f134b643ec4ae2be41b93b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/72bec3bf370c384bb936b274e7cf4c0aac46f573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/7785cef3a64f4fd162d38ed9940eb35890e3e05b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/7945894fb4067de4a43b092bb0e72e92689d86c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/7b084291ca0b545059a7f070788b00b6b1b94073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/8132604652c430496cb586206f222191f718c576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/835479a04a2777eb59c4e2c7b3b741cc27e182d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/8507dc1b3cd28dd9f192d241b9b7b7599b9bc1f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/8b72874de2083aeb5c85f86afc1dc1ac55f38bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/8ed9505a949b2531f9ab4de18f8dfc217bced80e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/9289cbcf54026b97d5de19cbcb787c87ce21fabb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/93281af94cbc9eaab5c31d228fa8a716bad39e03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/99d3ea2607ad1515b09e515ae86505a0c5705419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/9a421fa2b3f3298302da846e1c6820f5f0edf2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/9d7c5d8bae6bc6078d431bdfaf25c51db5748c8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/a0089a64aec3bd5086efec992d29e4eb2c3d516e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/a13fa8f3b424d81d5b2fa8e88e97ca77ade86b28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/a9d318b50117a3c12b2e613e375dd70b133b71c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/acdca6a2d1f42a2df275b9b382ae1abe4b5a8403.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b0556404b9cce0316d5bdf461cdec4a22c22b535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b2276c71840e1a3ce87324887d22ebbc761882e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b2d2da533120edd48c8e7a93dd176a291e4fa718.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b327c1fc1af46dec689eb9e82973bd2b8d2cbbf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b35c6e9e0a706128e787c6862fe02b0ea3f40820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b8bae29c2d37485f654c2a052df257241b6a0961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/b93afcf84fb32aecd9b26bf739e0d0852f36fc4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/bbd2ad4c176a70e0bedd95c471fc183ebc1e029d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/bc13fa1e79a57e7862c8ce44f955dfc6a114c92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/be7f9419381a03ae757a536365ce56e42bf48287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/c0502f5432b984c5aeedcfae9992890cd01e84ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/c98aa9bb8aa0e1524735401ea9b0fc443341c30a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/cbf797cd250f51127608359abb8b352aa2d79746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/cd49c6e741e933c12f7e218a1a4a15f9eb0ef61f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d40ec4d9f3d752843efb6dcdc8090af8b3ea72bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d46cb76fd0f4b0444a737b4d37c8dbefc18947d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d5f1e43e7eb126894cae3351d4b8ac492ffc64d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d66050cf0d39d2c0a46cb8b36ba91a400f953077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d6a88f5b7562e6e97620b821a5f5d938533005ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/d6c18c5b1c891f8426659921bfddcfa429104644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/dc70d58ec9059bfa5ecc8a59b2fce5c836921698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/dde399f0ca872063bc687ff02638b4d7e457dbcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/df592e189743c9f772dd6469a6390972c243ff3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/dff1bece45999c3b1b12b9886e4ac306e7dec2c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/e2cad663571c19cc1ace454e8df29dc9c2a8e8ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/e73baba5461219260a60714c7d1acdd5f2c16efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/e92a60566738ed51d0f2602b769e0c0fe38cd6fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/ead68b0eefbf0f4a2390311a760df3fe6cc848f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/ed05e703aff44f76f264bbe10db274b6f2df0a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f1dcfe74d5ad3cd8a9d3f35f2e38aa3318c8fc83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f3a6043ccabc7dce70162cd5b6ccfe9952a0b9f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f56b8711ec4eefb2fc4f985761fa1ae7d5221e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f584e897eff8a7547e6045761979aed13dc86581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f79b3c3de279668bf9bb848d18d432fdbb2145a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f84e8df3b18f46cb25593f0792c53543809e0194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f901112add25deba56d6805c4ddb20166076a14b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f9b9739e310ed873a4ddb9c1d15480f38130a98b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/f9fecb9e01c3d62356978cab4e51c09a46f89581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/fd65b778744d46e3e096631344e3a35aa3afb914.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-04 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-04/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-04/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2017-09.yaml b/configs/cccc/mixer/CC-MAIN-2017-09.yaml new file mode 100644 index 00000000..0bf4f848 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2017-09.yaml @@ -0,0 +1,761 @@ + +streams: + - name: cccc-CC-MAIN-2017-09 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/03971464e046a29fa9be0dd75c341a70c1bacf9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/03b9a931e85f397aa54f1ed624743ed9a28d9b4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/07477f8adee09eb7bca6cbec3671ff5e93822f9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/07c416da7f96a29860914012c79b7decc59e3ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/0c2d81e30bda5dfe40e881e6d378011577e33042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/0da68c09a216b4f168b343e79e27873629346fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/0a6de8e0ac19e7cb6b0d20d42c82ad9674e5e1ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/291e12ac82e461731a2b3a3b4e13d6b4cf7f8063.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/80a961b11c0cbd12cbb60edfab01393c51af962a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/c406cfb539dd7d777f59e0a8e4311808e4ae06d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/da160e9d9da89cd46c6b8a266a0401204a190dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169769.33/warc/f9d8b53c710d532711a076c9772d37283edd57d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/1d2cdd1f8508d9907507a0974dc1a0a69200b037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/1e27446b31562976878435cf576faff6e11fefd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/26de39ed4587204628d75b956d71c3e0cca363fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/278b80e05c14c0387ee07040009b15424e71e1d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/3dea8da1bda01a13d56458feea4fbb69aedcd205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501169776.21/warc/a29b0731f62fdad0099f7c232a0865e34b9aaf5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170186.50/warc/04176ae8574ea469ed60fb2d510b7d48e4c99369.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170186.50/warc/2c730c87014b3fe2a689f7eac91db0b2120c3f33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170186.50/warc/547cbacb1853aae7871d6ae79865e6fb0a6076d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170186.50/warc/5b9a15ab9f4ed2c290de9098f46530d556bcee10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170186.50/warc/5e3bc4dfed65b566bf5db7e13b5135458db9df3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/1e5735ec6ce9d4518b385a0f4ac7a59274e8607d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/88c7036950170c3bb30b03e82648e5222dc18327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/b2a7c31c31cd7dbf59a386d086c8d5f70843c0f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/bace42e6d36700fd3bcdb0ea3b2c5d1331ac811d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/d3c2933dbd4dbe4b60476daf81ade67b6739a261.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170249.75/warc/fb813bb1087f819c0bc21c8677888719807fb33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170253.67/warc/70f84cbd72b56e22eaaf2b59a38c67572dc02e1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170253.67/warc/9b95277177e4b1f8ad6b0b9a8d3128733e14c8e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170253.67/warc/bcfdc06f595b7bad4bf4189a4094e477370c9afa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170253.67/warc/f9ab1a0a190732d7c1c82467e013b47247e96204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170253.67/warc/f9e0740d611025e6e587db3d20f361da9f68b9e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/1a8b1d74740f6525d7ca24ac83be58da261dc414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/66cb5a33d2c8f447a7aee3586902b20d17e59fe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/6d2cb2b1aa190d9f4db7ff73fa13458d6eb2cfb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/97b10a91ea62859eb6e84532956c30e4e1526eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/9c4d7697a7d025fe2a2741e6e348729a6cfe79e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170286.6/warc/9e06deeefd9a5eacede2da54fcd9e744e08e1a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170380.12/warc/49b32a2f3e48e8d7a3f25204c09a9a8e910222e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170380.12/warc/ae48e1aa0794ca4ca1a357cbef5e3317a09b156c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170380.12/warc/c6dd2073b5fd13f40317279d6e76356197a64350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170380.12/warc/cc8958fa58ea56e525df6f4a49c2ee24c692f803.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170380.12/warc/faf89ecd780aa00b23289c047363142080bbbbeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/255c6e4dbd102b30ae92826391368c329c1bdc5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/5d1e3386299a7069fb85352ff6082ef90604f862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/adac68682cb1f5767d433ae37a36990d3e10e40a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/af29b1171a7e7f2062aa7fb160b9a47b275b7705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/e1ae05b6d553028e42ad3f0ba269711297880be7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170404.1/warc/ec46da2e9c8945e890915fd40373cdd5e803e8c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170425.26/warc/0c92407fe25be9176689dc2cf43b7368e559b43d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170425.26/warc/232e48b96f141dd94866d61034faa6d88ed2d58e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170425.26/warc/a3e74c32abd45ffc782f092e17aea035ab8b765c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170425.26/warc/cc431a36520fd45b03a578e8a5ac93f6fe7b6e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170425.26/warc/ee1fcf65bf4bc470a8e9a35d6d0a02e87445ccd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/2fddb468c373b4749c0b118118e0e64c30531ae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/654dbd18438ecb4cd50f40ff49b72c4924a119ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/a4f1a55643d9aa94e05ef2b5a20b7d77cffac182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/c573d782445f4b35650cf238080d55a00b95dc92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/c98b6bed78196a039d4852e9ef3be6f627173bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170434.7/warc/f1aa4aed755f63ed3aa6a0e8168ca63a5f186589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170521.30/warc/802a5d23fc0b8576d708ebe946c2c9813938bbae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170521.30/warc/896135646e5fd46ac3cec34d94ecfbc50b94feb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170521.30/warc/97348a5687ea01f49e98a1def5640a02891b4477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170521.30/warc/a234b30d83e5bcb01af204107068c199c4442654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170521.30/warc/a577754a4d1c570a5c3a44ec1ae48976650bd2c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/1eba0e33fb9d125bfc16c20c1d500035885e5413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/250a00532ad61427df45384225764c00ec4e76eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/6e93b4f0151fad4937182e843dffcd23ffeeaa61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/8197487465d870ef421563528ee26b467e8ee70f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/bc2aed1f79be806104891d0d7683e91dee8dec09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170562.59/warc/ceaf8bb0b86dc57a7430f42499003781ea9b2afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170569.99/warc/080dddcbc8f25675c55b9d7b68dc888b6fa22335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170569.99/warc/6eff5d7d3ea5a9b6f5748d243bd1e7bb094afa76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170569.99/warc/bedf728bb85efdc50554603419a997c3ff43ceb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170569.99/warc/e57e4ea982066bf415296b1a46353a4e2ee1aa55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170569.99/warc/e9fad6aa8eb860a4be1f5879eaa8cc2fb6336b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/237520be52ac2a013c3bf7703cd285c21d8147a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/343724917584d5cfd8832e80156141a262cd0c84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/3b95a347afad55244b8ff699064523e62af71db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/6eec4ed052280dcfa641b5ed5abb8d586be4f3a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/a5a47ca540c4d1fc1050b2f0948fb737993a44ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170600.29/warc/d40f04435178b5621d518094925c32d2a84650fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170609.0/warc/7a053555aa7a5c6f9b64e75935f8b85db89cd217.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170609.0/warc/85be3210bd49cc7216ffcf0582a616eab0976906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170609.0/warc/8cc85ea8bb17dba26e244512bb49697360cdc13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170609.0/warc/8f9193d84f17b63ce48a8b86f6c11936dae61d0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170609.0/warc/9dcd65d28a072a808454d908df1e39ff63c82dd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/5f6a2ce42b91d424dfe35e149ec00ce68f3b1d46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/809eb98546d2b3fc3ab02ddbd9c4e2609925df72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/8a8e9840797ffada344025dcc1fe73435f0dada3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/bd90d56fdedf35a2c7ccd60ec6f0410518d37e06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/c8b9fb4c894c0758d21b66fbd5f13d8743f45f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170613.8/warc/e72b59a5a72cd3760b5e7113339cdb1b8ba40ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170614.88/warc/0ff47b9a372858d478b80805f241e33fdc8b69f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170614.88/warc/5d619501b93d6a41601e3388c0c062f58549b009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170614.88/warc/77d0e868eaee35482f0a2478f4627519f11d53ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170614.88/warc/7d5450556d864ac3f3d7b720dc71cac4110f43a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170614.88/warc/e6f74e296b100f795e23de77f53de859351761cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/568cffb0de84baee03952c0727ae415e69ca1102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/bfd8be5a3a61cf8758dc63c2292bb331f8a390b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/cb4affb58ea14627a74f9c849b7b95075aa06425.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/ddc9ebfad69127d92746668e7af7ca0de80272d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/de3868dbbc467aa6049904f14db87455bebec830.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170624.14/warc/e5884cda745872196d800b80760ef0a4d07bac24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170651.78/warc/0da6d0cc1d34cc823de8ffcf7c31ab475e5d64a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170651.78/warc/42836c3a6b7f9b2573e756f5876547a638108074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170651.78/warc/90472a96b477c6e9dead512d3f1f1c1e2c185527.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170651.78/warc/95468f352f0766c10bde7c742a78adfe05bb37a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170651.78/warc/a6465f132335024d68bb87f3d921773d767e6ba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/4a36d9e6063252969ce09120ea74381ed622b99d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/7a6709257d7325b91fab4d561215f17fc883d14f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/8c712120b3fd6aee7701d508d28ade37627ebfe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/d2b07295fc64768253ee1978553b3ebdf6042e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/dbd2ac130ac97b1c730f87875bbfdcf0752239a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170696.61/warc/fb4dc7cbfc74672a955dc256c8f0cf4d8ad87660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170700.71/warc/4da71738420140cc70dabf0f96ef91d043388779.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170700.71/warc/573754af4b68feba3b4c712ad101208d0a1ef2c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170700.71/warc/723272ebc6b39cc6df2529683cfb6d62a6827383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170700.71/warc/cb59f3bf488b85796f9d4b82444a213457fc2d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170700.71/warc/e3d1ed78b87694c7ec1e3d887406b2f065775477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/23dd2a2cbfdf2787db4b72ee94daece030ac52e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/5c3fe26cfa316dde39e1e8573764164282971a3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/a36ccb5f4d798c8ba8738275857242a2b7deb959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/d40dbcf431588485996997e7aceb95b5d5474a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/f7b90c87b434c3591816be176cec0944fe7ecf78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170708.51/warc/fb110036af4e12dd8755c8e2087498eb1e4c130c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170741.49/warc/06130c8bc9c338c43845844cfb6bfb59a41b6a32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170741.49/warc/1522774f250541083b64b7eb6a63416ca3ba06ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170741.49/warc/7fb634962a36479a8105a56bc732651047f68cd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170741.49/warc/bb45b46ec9ea1b4d9f5a5925d13a027b4abcf092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170741.49/warc/cc57ea3849481e8a4653120f21fdb326fe58ee77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/0449bb9872840d0cbb92c57fce914e89a36a7c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/0b755680142d43e448d1992c2e351cd4d08415df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/22a5b4c2d07f802e527e9fbb61fa9f509ff21722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/9bf45aff3dc5b4e4672d1f1ad39b5def964256e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/df03dd97815c9e3d10be8e7bbbec8803c41eee82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170794.46/warc/dfb5e05b4d920ccade8e4183eee060daa7907dde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/342c9956630b55111f0d27a7b1e471c96a6bf218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/38c105c968fd43e16e6fb524a03871ec7d35e94b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/3bc45b66cbd468f517e35bb31df2be42933acfb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/98d4071576e9464dd09e26bf9645125d05a3e6c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/b807225da2059656b17f8ccdb99c4e4f6565e55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170823.55/warc/f8f75781b58c274b3af26a3175393454614d646b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/43e50778f11cb20115d28ea040e9227b402c3b71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/6a243828e01ee955a3b737676401dc411b1333f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/87fc8edf9268f27983aae01b23b3cb0d4ea0dd44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/b7fab904b21a59684000ba886892203f60545160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/b8bfe33744aff9b32be56333510effff26aced6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170839.31/warc/deb10fec8ef15a1b9dccd58acee4692a15eddab9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/97f0603dc08a590110f9df9ee9d32c00b47d23a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/98cbd347314dfdbec726b372f8112568e8373da4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/a14b6fec703c56c694c4d03372c97c2edb2043c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/ad7e4c66a7bbeb2aa09bbee41532410c14916ee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/be5a1b1aa6397e6fd1d43417522ae7c370852399.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170864.16/warc/d90d8e82f2fa9b20209a5c2133572effc535392c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170875.20/warc/75b9b3fe7d5ec292439d1e0520c669e04207bb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170875.20/warc/7674f0a10ce3d1b07cd5cc8120cd2604a05f8205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170875.20/warc/7bc193f672e55bc40a51ab24064cf20094e96efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170875.20/warc/acecf7218759c52080b49806bc73c49de40add4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170875.20/warc/ecfec3bff07b745e861f4df9426ac5934118398b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/0247aa34377139289bc2e4b7814f02a2f0e42f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/29badbbe4cfd41fa9204553ef6273969d6436bd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/33a3b1c61d0e05a2ae646689ebec3e733456b599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/893548164b4b6d05a9712e9eb0de347b6c1cb604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/af271c6b2a04ee943c201b29428868530eff5be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170884.63/warc/f7e06de39753ea7a34d70f80acb5b3862be91f64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170914.10/warc/148f4b9f26a81930f37eac4cecb847f550f8a462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170914.10/warc/3e1da4c5396d6f66dc1d4111d25b3d232cddd8a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170914.10/warc/9a40ad0a68c9890c1b33227f858c60798fc15735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170914.10/warc/ae5d911ebe14338add2b15ff53db93cb3e507731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170914.10/warc/b61b723df617a35c8a0fccbf2693126755c64c61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/2917414d1aebbded10d2911475cd4a9095057711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/3f79f65059d632aaf96cb12553f66594b50124d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/57d74a7af8a0a8211c51f29c11125752bf710385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/5975d5ca768edb93cc04fb9e9b4d6c96487854a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/5ea3d32e00bffaf093238d636778d5d595a1fd1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170925.44/warc/916c9195a3de6329036e2a48ad264752665d4fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170940.45/warc/0ee091f72e5f89224857be52c63078f34f0e7c95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170940.45/warc/2b67c67d21e66b0e3c42671de8db9513accedf65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170940.45/warc/9ebf07d6e346b69559e8f2ab0b040eb03bafd334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170940.45/warc/c8f9069120bb0e69601d7ce16481ab2453d48126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170940.45/warc/ce091f289732113329fe35965683d69e4ad9bb84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/224ae118be29048572dcb1774cf6f5d99829eb2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/22fef17ea5ee63aee5955e1c998ec037157d27a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/808f105add8ecda8ec8493532937229cfcf6d47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/92b94f497b7892d189686ba05a0690be17aa8808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/c165d184fc4b550149fecb8ef8cc3ea53071b73f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170992.17/warc/e339ce55c4ecc2feff0813ec7d2119d8fb164438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170993.54/warc/0497e180516437510611af914d7932cd5694ec3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170993.54/warc/124d8f0cb41e9fe6af37b11c8121f904a84eda48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170993.54/warc/73711c26a48b6c0123b4283d5cd784e8dc34cd18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170993.54/warc/74566ed574426a199f3a222d8bd44947b024c2ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501170993.54/warc/b021e39d6e4d105f31469c7fd136122025b73cb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/02839dae9d81042974321f7ede2dbbbbcd28fa0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/205c32bdc94e3047fdf985bfa9084473fb8f9d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/773272e7e104228d4d5070014f45162fcaebdc27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/9cdefb6548f51d290f8c51f895ddb3641a5eae42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/ccfaba416eba2de797ef78b906702cef794d5e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171004.85/warc/cd02975087bd9df3a7bb6f52204750da756660d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171043.28/warc/330660f30b3ed1ca5acb642b771ae6515aa95495.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171043.28/warc/60f8d3f7daa023369a2a7b222e3430ed7b3331c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171043.28/warc/635b81af7c34dca67fb91c8856c15a808c570896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171043.28/warc/71fb0593f50145e2ce84408461d086be9c853df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171043.28/warc/9972a8bf632d042f716013931f3e5239f1718671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/38d228c70a36dd46a709001f445115178805d219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/b0a2ff143a1f9fcbfcb488bf813417d70b431631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/b41ea0be7de1c02df133a75746a5453bac4bdbb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/d9c56a55d51955e421c5c02e74c2127da38f28a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/df73d4ecf3dada3243acea5da2f77716d711c18e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171053.19/warc/f40c1f0a3b5ceafcd88757258fe418284d480bf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171066.47/warc/668f081dd5c7d92b1c921292da9f6e5cb1bc4991.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171066.47/warc/b9d47483e8c20c8fe787c0ed29d73c93c7ef6e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171066.47/warc/cdac752fb7f3edd512a33be7bab09284865212d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171066.47/warc/e30d2af2fcc1458ba95d1647c20daed7b41a2291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171066.47/warc/ee2d1b5a4921ba99196040264da989a9ce684dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/20334c94d11b2bbdaa8878a5d976aac5023003da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/28ce2e42979da2ea32c35f4b5b3be401338d4b32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/85ce92fc48dbd62600fb9e342b37921be6447a7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/9efe2b5dcdab06dd2cc9388f908dec5d4bb9f7a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/a23445dcb0161876203c24f90dfe35ee43760d21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171070.80/warc/d90df3e6c232930215d75b0cf8bb72513e490cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171078.90/warc/3f66e59e7ec1fd264250202d8020802c6a513ac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171078.90/warc/70e74a426ff82e16a00bbcb1e69d938d0e4c2eb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171078.90/warc/7b8f4e93d47a00053e9fdfe69f0890048f9f4338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171078.90/warc/96e680d1ccb5167f6237bd7ee03276dcdeb4c888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171078.90/warc/c3c51d61b97c088cbec6ad548369dcad83267515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/6683cb7e9674dd179ffccca7f0f25a935570b924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/7d677ddf307b8c4867b444dcb22ca5ef2b608532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/b2de77142842796494dc75d870c67456c0551271.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/b581104fe2ecdc0203647616ed2ea79def113417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/bb90da90bacc112b1dae7ea424d221a248278d70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171162.4/warc/d620243af7d90f89a8146eff7d6f248d4723ea1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171163.39/warc/0e838f18c2c68d8f1f5b01cba944723ec7ac2779.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171163.39/warc/2326ffb4fe2830511e087bbd41e41e3438f81c85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171163.39/warc/34379938b947e99010be00f2d7fde87d89136037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171163.39/warc/da35a33f0e4a905e69cb55a09657c1f7a370a886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171163.39/warc/ff558fc15375fd564f3257ae4f038b36460e74a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/1bb54de40c4840144d16d059907adb5b5d81b176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/21fbcbb3670a6efe16a4d33eb4321c2162d07886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/477998d39093d9d0eac8944064bc2f9ddbe06059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/5f08551b3d5fcc2917c4fb38e2515020fd01bfe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/c26222afc586aee47230c9c061f3b09dd7dfe88c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171166.18/warc/f119b19e048ff9cb4617384d2e4a724969457f8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171171.24/warc/2e6dc1ba988bf30a6bc930769bd58a8649204538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171171.24/warc/417b2d8e1c94effc1e5db1ead77c499072a54065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171171.24/warc/9473b788414ec572f92f9cbed063f793ad599568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171171.24/warc/da91918e2a72b8ddc4e44b1888800da9dae7d3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171171.24/warc/fa0884438c81e85d8490d095408a72d280515436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/17250b78214736593d09067c7405a8e8d2c1c946.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/9ad6a86f5b5e1512c1c97eb0b78230c33fb50269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/ca0eaea9534c86322a173e5c5bab4332a0e4decd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/cdc70ed6ece70d06daad1cdd1078cafb1b71cc88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/cf917070c4f8f09ebe208ccb50d6fdc7cf21de17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171176.3/warc/fb24a6d5e5fed65a054e5ce296e2430fca17c28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171209.83/warc/158e848410c3e301dc478e591fdb71820da897c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171209.83/warc/3294956098335e3fd58e6b64bc8a5c1d399cdf8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171209.83/warc/38f1974a0c321256a657e12a467e4f1fb0cb355c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171209.83/warc/be0a409ed71e10be9c78ca3a5c17136abbe747b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171209.83/warc/c64d28365e54d838d70990fdcdeaa008deb37707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/0081c9bd207bbcba25da28dfb238d124ba2752a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/93b65d431e2a0020251b2a44722a6feeef30c376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/af23b48066c83942bbf4e0fa22275bb72cba75e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/b7c036486ea4ca28fda5ac5f41d798cbb779a0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/e59c459ccf827e714778a720a2a0cdb5a90a1309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171232.43/warc/e8db0ac7967fff59f5f7c13ac3e647c54857f35d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171251.27/warc/2fbd548c638c974063650e86b4702f9db01e60e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171251.27/warc/78af1d7b0df2a709a8ed7ac7bfa490ed36a3b90e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171251.27/warc/90cdb32070e6a77e6a6b689884fd16f54131be60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171251.27/warc/e5fd5fa96810900f47979b79ef4edcc19892e383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171251.27/warc/f86dd7b9b8e457a42b00c3c84ab2c3b5e9ccabe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/041c562c75c8d29fd8e9519f2be7106d9b0e13c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/6783d566c580d8d3d08da1ce50beea7110366760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/6f9cce2dab110d99878a190a00ca035a3a40a085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/88bc5217e2c631f2877f5fe84a8bef87ae4f25f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/ed4d2eaa4581516e10f6bf2abd24d798fb2de9ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171271.48/warc/ff9b44f931e69b2c53ec409852afda16ad13a9c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/0296a76b5bc9280eea5f21c30aa7d8717623f8d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/2762c5190e9020d61b9035b5105593a89aa4c34f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/33b936a319a0a6c3eb05bfbf99629ac66e69223e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/d5cb25cf55b128041d7ea90bc9e723c929bf6cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/db7b1f0cad1b2dc6991da40286ec7b4d9ba92f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171281.53/warc/ffdb56214e9dd3c1129ec81deb12f3c9cb5db998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/00980cf61baf111ecd21c69485408151aa9582f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/21d1b24a1d7f71a2f19dc51fee0d12ac570634fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/24c8e34ee8786a1eb7ab0eeee292f1ae9dbcaf6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/62ba5fda22e9a44e32c8e041145cd37cac90c919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/96218586b6288b36c569662a1a9cac51d7f1746b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171416.74/warc/d4e5708fb9d4cf18e77055fccc544723066c9c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/1fe8307f27347766b32fb3685e6ca34c822306b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/27d32b1e5178bb7b841c8cc536ea444fc0879c0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/4d8a6dcac2c615536d52a86719e6bdadfabe8933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/c58165c0ba87f00804c63994d7613ff55569041b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/caf97098e3f4c5185b3228c7b41bc5853e13ce0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171418.79/warc/d732543bdd884acd2b49c7ed8e2677746bd1376a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171463.89/warc/170641edd2f9c7532a3b38b71490762318eb549f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171463.89/warc/901c17ca13bc815ea2b4cc040eb6d515a8296307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171463.89/warc/92b1274c99b26ad525bfea22fd925a9768fff3e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171463.89/warc/bf060e998d0bcf802837181fddc30f4754445d53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171463.89/warc/c2ede3619f4932fa571248e17967b1f52075f6f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/051236449f81ba2694be8ee1d00aadf78dcce8ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/2aed144d5d87120d16ffc5028338be26b3f17894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/9ed2643163ad7774cd26854648daf6fe20b90b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/ac9afbf55776aca8020cd8aa0b3f12badd085f9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/b2dddc8f877d1cf4225bf4e326deb95bf2d21c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171608.86/warc/b33a860bcca1b336e1acb7810082925e42380af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171620.77/warc/09184621fe21143b48222d71342e6a7586e723d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171620.77/warc/5459bfb9d1e645aea69d69b1c093956aa203b8a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171620.77/warc/7e80efd3e6e5eec4f3e1dda6d74fab88680cde5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171620.77/warc/a6179fdd971814087eba9a5088c6c132e951f8dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171620.77/warc/d88c8383807212008b1c31062d53ddaf28413d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/002572de4efa8805597efb92e6ce8e801146fcd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/0e34fd5dba672ad9e5a50f53c2837c09fd416ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/3bd071cd4019cc46d534a49729086e33ca86880f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/436f1720de1e5c4a058432621a50239398be34ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/6d9595171844b62503a54de94367f66345717c9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.84/warc/ddb45bd15cb8f5a38dc60b33f25fbe5a07947a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.92/warc/058bc581af3b74622e8ab2c5c779dfa95da08989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.92/warc/1bb1c01f7eb44126bb6728620987b2b4f92b4036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.92/warc/48600275bac3bd2d299baa734bc12c2adfdc3213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.92/warc/c8d18ef1f602f8086075439429cce5eaee36a4b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171629.92/warc/e10449783552f298061076de71eb382356ed4fc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/77939acf0bf2b972434b4ba34170367c059f07e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/9556d1b4a8edcef6ed647dd4751ab6ddc5b16ab2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/97d7d27e302fb7ebb7c23434e71e91ccb6429b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/bc3e5955e1fae6f2a0af54d1e3683f1c15acaf35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/d7a80c1a814fb5bb03b33bf6b83c394d36955771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171630.65/warc/f6ac94192c11ff8edc31b9c2640a8e0540767a92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171632.91/warc/2a93eb1ace44e15648008b14ee500ff5d8f17109.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171632.91/warc/80ab2570c9861ce0e358e6af3a6035e1fec22f7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171632.91/warc/8bc7936137335dab76e62d83373eaf6a4bc8a902.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171632.91/warc/d8a6b5e7720b3e16b117cf020da7c43c01c2a46f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171632.91/warc/f7b8eb3753b6ec012f81f59952df7b6b325261fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/1e82e59f5b23c0a2392824f8015fd0538c0980bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/5920a0ca3efa9697047c4cad51356d46f9880928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/7a8aba071b0b8180c707ed40f2d1b94f7f06db58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/7b8c2982374adb34ce90c8f431f65022c684cb67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/7fff019ff71e629bf3d739bc57c8ff1535be555a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171646.15/warc/f351ced98d996d8ca7e41445f1b907e4f4dc5d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171664.76/warc/013fa9fae71c5da2feef20503b8b7e99420b4e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171664.76/warc/140d20dcb1afb03fd5ae0a13bf9735735a5393ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171664.76/warc/2198a14675e89da3393a21e55d8060c6416d54b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171664.76/warc/71c6ddd8added883eba45200815e1ff0ef39a169.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171664.76/warc/ac01e74481f735bbd26b70ae8d2e1ff69b8cb8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/0b5e6d2c8fc8c6be25c331c4027a0a8356ed2626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/21e49d16917602eb17bc8eb90a650d4de40aee78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/55b4be41f90952cb034de5181c1443e5d4e9c9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/84e8d7f10cd8b884c1e561ad50d0220b22f9e187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/ba416f4fd8126612891898f082fbe6ad11389e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171670.52/warc/dfc41b9cf6f604e119d8275fe87bf5f9b40c4bf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171706.94/warc/32dc72056926294df9f242d8bc75d73b9ff08619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171706.94/warc/47b60431049fc6b47f0d7df7a83e758b7a9b341f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171706.94/warc/6f24dff854b8c2917e9edd8fce782a18e0c73df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171706.94/warc/79236f4c774682867008d8a0bbc1e2776031fe21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171706.94/warc/db10a4a34d24fa3835096166c7839e0a47dd9249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/10b25c35287cf7779f6b9b72fee5e437ae686554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/1ce2648466c755652a663a42a8057d6d60d2804f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/35c3c451a5a01e43966b6530b8c3f50e97b457cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/5d647288fe3f549238f60b8f0f9d0300dcde9fca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/6194e3a1a8be63a07ae4e32a49fe325466cf9482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171758.35/warc/e7ad2d942c503452a0f79d1e998814c22b1a0c78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171775.73/warc/19e083716879f8b0f428736223f231565f807cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171775.73/warc/24f0457e2c172a185729c524df441687647a1301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171775.73/warc/d4482bbe292fbb825599e9699fde11b5437b6002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171775.73/warc/db7eb5a1260274519c4cbc1e45817abd7ade943e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171775.73/warc/dd7bfcd9e5a139aff632a1a221d5513afe4ff60d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/01af4c55754316de8eda990c77c0ccbc876449a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/50c72f9a2b3cca29ffa8111ac665282b0b509435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/d13ff840988e0d7e30f084b5be50c9e8c534259b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/ecc5bef9be4d8a2acaec038c2fc95883790cd609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/ed6a5743a7b732a48a597796de75c4e13028bef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171781.5/warc/ee58f67d443ccb8cfba66a3af8d36ffa58154350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171807.25/warc/122efca0ea3d3d4b973f4fb3393cae596ae19b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171807.25/warc/12b14fdc5c911c59529795812775c0de232a7967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171807.25/warc/1c8ecadf9f7ab8c3045dcb924e16b3897c7080aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171807.25/warc/21c5de99bd176c72746ee977a6c78d272dab8fa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171807.25/warc/f8799cf63491b3af33e804925fa70e0d60b91a32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/2237ae295c66aed3899870d59ab8af1e5a0175a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/41ed5e2800e932c55314edb1a52754d658e5fe8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/763eccec14e5218915cbe1a45d82f8b272283f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/b8240456106296a309c1040a5b53f9ee6a6a7e3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/f6a420389892ea0cf89b8ac6451146ff419c785a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171834.68/warc/f72974457902bdc6aeb0feb1b7b7b9bdd495d028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171900.13/warc/00d8892ac547d954e2b021156b44dc91429ae440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171900.13/warc/724f382b575da4035de2695a76c4f6272566ff11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171900.13/warc/8ea420300fc8183c06dbe593364387e0319496e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171900.13/warc/9d2a91e6f60cbfb2539e6c625354fa323dffeeac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171900.13/warc/e860dd8c451a8101219f57609f6800398dbf31ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/16ca8ae3307288e69c962d8b67c714bc1136bbf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/33aee1dd7f97a30f0f9655424c84bb6f54eadc95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/35aeae2e9326d4cc4c3df95cd5b2696fa3d4ac80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/578967a586f65e5439b44ba32f4ab2c8e2d4c04f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/a34b290d09bd0fe43cd6dff926d95ab7bade0ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171932.64/warc/f1622be1a34ae6cace271a93b0eb0c0a69450658.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171933.81/warc/21e101b95434cf51c5c1b9651921e45f3cc78610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171933.81/warc/296d3687dd9a899495acabb7a1a94b4a2c377427.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171933.81/warc/7320b81b27d302b285955781e804e818448b9599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171933.81/warc/e2f85fbe7c5b0a84388ed69d8f73865e4124dfce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171933.81/warc/ebfe8270977116779b65ad9216ae7ccd761b6ef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/83b0059f7e6d112402e5761740ba473193b697bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/8b2bc596037e2161f0f5c9315200d3cbfaf7dcd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/8d2c3451a3b887d049abdb11696fdb9811d1916b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/a9c343b92fe58cba08e84026412dbfdd71be394a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/b01697fa2eb63f1d9fd827ef5196801609913926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.2/warc/c96c47c99138ae5fa8f03e9ced25b5ee68ff3a22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.32/warc/5d32a5ce1bbe9e24dcfa6fe8c86be1256da0e92a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.32/warc/8918575062dd4510d099f4ea59f5e93f9a120d7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.32/warc/ba8473a61b938028ab6d1dfd27602dffebf0edc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.32/warc/c00081001077e314519e7795b870c02033fbf533.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171936.32/warc/d0b43a014bd32dec19838d554e5c18ce9c417685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/20bd937df514b31625b51e266c5d626fa286dc0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/6fb0bbcc0d7a18a703f6a261ec03d2fedb0f2248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/723a3ba27d9d016ca3cf846ff40e2c8611de0da8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/7885ad61d20332a96378031002eb28dd3c302977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/b510fffe8ccbf29c5ce16c913fd2b3654daccefe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501171971.82/warc/e24995d9cc843d123c391100b03a2d68ca7ade3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/495afb709cbab2efbaf366a3bf65fb5e522b59d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/7b7b88b34cdcb81d3a017bc3f1656adf279db389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/803a685732826c42242915d672c38f5d9b4f5706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/b77768762a1f0a31e35ee3343834b01524d23d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/d8dbf7781f4897e1c765c26553662056f4775bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172000.93/warc/e58cf595f49cc3176fbec2c3b59719ec7c1740df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/18aedaa1481e4d3ed6495dac9755bffcd10f3b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/4390bb6d7a9b769e2dfbde358661345c689b1737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/443c3129b17ee836fc1b0bff3327fea95da58362.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/5fd9a88e8e7cda07c6ca5cd3fafa00643c7d9352.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/a291104a7266e4d26e4fb4f73b397ca3198c0543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172017.60/warc/d9241e92c009c3f8cbacff92d06709ec2b1dc718.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/404c2ff7473eb4afcc3d2959a990fdbcec01a0b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/5a32d376b00fd6b867249715fa278956bb6649bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/716554732016b20f9ad74146f1cc3c1e93901f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/8cc6d7af7384f4b3a8c02939fa8f8c5b07a73aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/e278600d4bebec1b82e1f4d57dab91ff2752ba04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172018.95/warc/f485345437a6ce1c49dd3e3efd5d235f9043e54d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172050.87/warc/3f61b78f468357ead9caa5b51ce27b5018d46e8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172050.87/warc/474ce2523a11c53d05d7e5580b551c91295fea72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172050.87/warc/489eef3ec6e436a0ebe2e6ac5475f875365c5c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172050.87/warc/6421ad1ecb5ced5d45c67b2eb4b5be47f0a78b00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172050.87/warc/fb7fd815d4a3c50dd15992e4f1d2d0ce01e53a5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/83fcfa9a8730dccf604b54d1c3fa1d413a6e4b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/890185ddc9c7c1e90910f6e230dff87d16d7dd25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/a37462b15724f758c01df7847628429d9796255a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/d6bc2c2fd5e5c3fb555741fc16dc99342135def6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/ed39ad8cb34a38e99d1c1fe26e27eb71c64b3a90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172077.66/warc/f422e3788f458ed5b4457bd6263bd723add21d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172156.69/warc/2077476bb6035ccf158ab5d64917b0585d5e128f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172156.69/warc/2964b62025c29aef1c213e7d7e01bebcceac5d93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172156.69/warc/2e2011ba689331ddb7a27d27b33e0127789e2b96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172156.69/warc/6d078761ad1e5e356fb4a5a9a8e12bf7d817c35f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172156.69/warc/95d081937cdf0b7bfb21195eeb50ea0021f0da63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/03e9dd85daba410972a9951db83a39434e8d764c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/0ea4c71bced8deeadc8eb4c57171c08fdafeb7ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/25144ef0562e102b4f1fef37c87fe9baf685800d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/4e0cd9a63f9c673f0575fe0d804b06eddf8bfd8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/8c38fc21cdcc4f085aff5c4e2a5ad3814b5be913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172404.62/warc/ff0c99ca968ce91fa2a9646fdc3825679d6c2829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172447.23/warc/0db097c426c5d4f29819579b89bafe1a4cf19d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172447.23/warc/214b0adc94d276c1c469911b3e9108235f3c382c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172447.23/warc/a0dc339f8ecd8817616f4ffade150420af2dcafd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172447.23/warc/c5bb089f39058c25ebd8b47b7f8a515332e22125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172447.23/warc/cae1ca077deeb1affad88c4f2bf4203fd081208e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/013c898c9cd3bd7e0868a84e9437492bdb3961c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/369a4c82f6994209e5b2e8bc82870620b83e4f60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/3be06514a15ba84e8f55c751551b2eea841af0de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/4b62174a98dd2bb5f73439baff94d06948272d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/7e93ada881a442f9081baa9c4389ddf05e017ba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172649.58/warc/a075c0343ade0a3bf1e12b65ebc972e6b1574ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172775.56/warc/0d72407d18514898d8c1961377513bc5ac054625.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172775.56/warc/2db2a45248486a973688cefc730f8492fbaee117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172775.56/warc/4f4eb3629c5e00093bc29e48624cce4228185ae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172775.56/warc/a19aa25c764843c38f6ea9588c7668c51f0f2038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172775.56/warc/da42087b2d29983fb9cda3d066b49d93421a2ab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/0c89fe5c2d381a8c56852f706b43016c79dd2885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/a96dcfab3e71a0db53556d3ca9feaace29734cc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/b0e805351d3abb7b199cd32ac72d0c579ecec500.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/e13365bb7e52fc20bc1942d1d30627c0d9a411c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/e180e249bbd17bbffc2ae500d785efc11100c55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172783.57/warc/e73de20132496392e5fb1f2c69e0e0339fdb5fb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172831.37/warc/0209f3331ac9e0cc63c82d56529332e6bb7897dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172831.37/warc/188be288fea07875c98d71cdfbb3f565b4927d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172831.37/warc/2dedc51959f47b649411bb5ba4c14b1cbf484e0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172831.37/warc/71c8c9ebee1679a0d602dd7e6330a2d728a59e6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172831.37/warc/89debd00a8677c86d66ff05c7d571ca23cdd2583.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/06d536cf764a4de1a0ca94d9ca0e72c1722c4491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/2d0b9e90954ab587d67696ba268c35d75bd747c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/3309d96bd40b37af2954940ffe3cce52e77023a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/620426455cec6c1847045ee183539785e7e93bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/69c565dc14e21c68d4615042f51a06d0de5800a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501172902.42/warc/6a1422d2bee04839160d5d8bfbbd657a34f489de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173405.40/warc/1547eb73a8a76a66a5259ea2fb8775a327ecc725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173405.40/warc/1f2f87018f7e12626142bb78bbeca6e530adb9d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173405.40/warc/2300e3a12052d4ab13ce69b24fb397b3db70fcf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173405.40/warc/7bfb464f2dbf56f08d967c86efbef7682303f5f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173405.40/warc/e949d7d6aef459958157a6decd91ec3ba72d1aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/03ff059bfca2a1dff2f6645b6f233cc5a4656068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/202da6711752239e6d59325a6f8166aecab648b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/4ae5b9b5d44a135628b17bccce6f848a8b2c1035.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/911bbc28bb1ed17f4542500e93f1d23a9975bf67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/da481a9175101c6a5fa3c5f6611b119e206a6a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173761.96/warc/fef953749e1e7bf9e713039a73b49e0263a53942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173866.98/warc/05e7e1664d0ef81f565be1a2d59300c38e26cf2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173866.98/warc/18ebbf1b1c035a207e0cb127b1128847814efb83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173866.98/warc/49f3a1b1eea16946cab4b732add39b75776b44f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173866.98/warc/d59703c5ee833c75a9eb0bd90221393c8d6bf33a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173866.98/warc/f27b3bdb70bff8ff5e7a8724bea7c2ffeb7413dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/0d34f7aafc19e244eed14d87137eb649c7fec1d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/3e26e076785099d1f2da423f3429c3c6e70723ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/4090b26d98efa0ccaa0612fd6db56e3d648d2dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/de8ac58ae0386a99e56cbc1c8c69f483fd4972f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/e810e0458b98fe651225cf0db48545a8d9104d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501173872.97/warc/e9cc24d7b4b818d0accc065828084ab1cc247c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174124.9/warc/5a4869af3c816a8d1e240dd562ccb3c8e04f7df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174124.9/warc/7605506a88aeb145dbbd1e0ff76f8a9ac4098fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174124.9/warc/7f72fd01ce11c1ba8bcc9dcc3cdad01f583b5215.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174124.9/warc/b336a043c44692a1386dfcaf89419f1108bb775e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174124.9/warc/f912edc9a332ab87024de5632acbfb9fbe2aa13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/3fc7580e796e4d35ff800c00e3aef82005402491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/9e4a5a58c52b3113b0d6b9820514866c523f5d50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/bc7041969b6c335f2fefc8848ddfbf4cdca99c4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/f5c1629b5356f35f77aa3f9f09a53d2eb2ca942c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/fe1d26256c836433690851112d886867aa95a31d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174135.70/warc/ff17918ca8fe70fa87751aeb0c0fda1b7e1b9233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174154.34/warc/2b57acf89badcaa7b3917b42f22fba23dd42b409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174154.34/warc/4c6c191c08e11815cd1f6efa8263fb5440d50cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174154.34/warc/7fd51cb8e75737fc2272b108aa5888c9bf28a350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174154.34/warc/87d90bf6f5519a5759b5cc5e19058fdeed4c1be3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174154.34/warc/f294fa18752ef1ba28895175473f11d5d0479c26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/0366de57b8fe3bf78efa13b3d17ed503bb723659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/2be412a3182a5b7aa7098b9d65a03ff40089ddfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/57d652be357ee2540a973acfd6e07ff8826d713d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/96ce81f68b88d432afdfb53655908c953805e93f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/a8b95f7367755558fcd51c7dbdc4d24d5ecab2ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174157.36/warc/e9f1dee97631ccab7f9a449c0d26b10bebbadef5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174159.38/warc/4ba0ca0110986ee5204d15becf8707f7eb4a75af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174159.38/warc/8e94e7656f908511e900144c9234525c29b45c71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174159.38/warc/91c9da6cad1050356c4e45dab6342229c7fc5e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174159.38/warc/bb450172a4e1483636e3c981c1f901189426f7a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174159.38/warc/c2797e056a7993ea7921fac84c7f9e6851f8c011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/1b29d783e306313578c8f67d65a310965859e17d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/45b8b0ed6453a27d54e84f502078d1779605fb4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/5a9d7ee0e78a75db296ff62ad383fef27636087f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/805d2ee284a8a53222179c146b8c9c3098301b44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/9cfc750028f6a444c545405fac661c4f22a96f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174163.72/warc/f942c214955ca5d206d6d4c45ef1e0f80fd5d52d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174167.41/warc/029ae8a79eb7dc7770fa477deffd4ecf75f2ab29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174167.41/warc/3cafd9493e073a5171f43abaa911434f05bc54b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174167.41/warc/60d1db74f75cbc971031f1d1ef23be4f86f1f4b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174167.41/warc/815e88e3d639a4a5046955ab8719e5aa2270e323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174167.41/warc/b9d18e88398e2cb42e825e1810dd340ca41fec4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/233db9aa163e578b64521d0c79f9e478fb93ea32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/3207c2bbdf8224fdac685662cbd44ec380c26145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/41adfabb024cb22e4e4483db9f11069bdc4bf823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/66a566ac293c1fe4755ff01cea47a4b8d2125432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/9dc081bb13003f9c840657c60c1924d21c2852ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174215.11/warc/f221f66bbe0c674a8d1647b149f2b7f513eef242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/1a30068047a21f7c703801600920eed1cc83be5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/28086e1cb54b3abf1cd6d56bfc5540a5e519fcd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/ae58f2c7014bd9ab2f6146147504111f77686df9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/ca2b58aa8c006b63e0f0675dd41971018102baf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/e0ba5ca02c9f3f1d028a154f27845bbdd023cfbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1487501174276.22/warc/e32c3f80c7fb81ce518912d39dd2cc40a026ecaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1912fa9e59f6557fe34727bd24f2edc108fa16f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/1b0ccd301d6b0fa1265227b1c79375a9c65de615.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/205912f5858a42d3cfc0bfdf57c3773c8ab6e748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/20747ef63a0cfcfd8b62bb917ec7d773f279d040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/237f736db511d403e57df0ab74d4d4835b125070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/245b81a4746801d3cd370848c089047b5a52ec35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/2a2cc1c4a0f3bd3c6e4a7dab4c756e9d2948841b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/2d482672b74475750650f163064c866c452195eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/33ee72f8ac61eabd7c55c853708c860e2509c22d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/351f86321cc6f684b8f415b9d71e80eca6135ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/35f5a7280e8a07aa042c425b63a5760f45a0058a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/3c047078b123567c598a659288e6dc1b22789d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/3cf04673feaee35524d9b321bbb0e3eec78372e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/3cf91c658c481ea85bc0b12e72f8300ed976a540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/3d0cd2770cc722bd5ac5c8638ad48e2e92e94df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/3d95cf7cbc7ef3e389e04d57e784b64aef97507b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/405a41bf5a5018aa758fc9608f9d211829c382cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/44fb2985c2c86492fa0583393c5496698e65c4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/45020a25c6cfc21aebe8806fd82d7103804f058a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/46f5538c801bbae79fbfade0be9f6fe857d60f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/47c1acb966ae3f7f8f2270de3e2e8dabd1b7d1f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/499dddefd28ebeca7c3168ea3d64db8cfed9e13a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/49d6a6a80ea547f6201fdbbe8398d0f697e1cd9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/4ecf96a00640ab5a2e6bcd48a7be73f6938ad03e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/4f60ad2cf3efff56cce717b2795ee138756e7e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/511b4059f810aeb2d8f8a571509b82774a8da506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/51d2480334f28a216e005b671d8f068529af617e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/524270a6fc8cc2aaced2234b49f113b8586b8082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/5271286faf361e5e51ce0fc6e54f8ceef959c02b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/59805694a5e99ebbc65746793e14669731a409ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/5b29210baac6d3a8e028f76c24595ed10db9760f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/5b8483b5cd9d1a2daa717632c0493e1c650cbf8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/5c40a013df8b72cfc9f3381d3486721e457c5b96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/67a0948e0b0f048ceca98154c7da13b0a167f65f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/68be68a4b837a8e4efac2636c7b40290da05bb7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6c4064b8486b99f9a1cf4946ca4b7fd79a88c3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6df8fb7ae9b66986df150c8a6838635c760641e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6efc7bf45054dcec05ea3f8175393d89929449f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6f2cd9558b0c0ba56a3eff13eb6229326661d55e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6fafc8c62d3ccbf71509837941a249f55c353f5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/6fde37c84a7a749f3689997750def979c6f7a4a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/7099fd77edfecf2d181cee920be5f4fee75da7d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/729dcdf1c710fa3de690c5f1096557a431d44716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/7d15c3b767136f79a6805029866b6418f4829d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/7e636f15908d56b41df08c34cd4a37c4ead7178e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/81e3e57a6247de1a3fd292ec5a325a7001df5de6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/82022379cb10ec42772e69027931253cf7dc95af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/8464d7855acb6e79475dceffb4c8d844e139136f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/85058f047acdbbe8afe2116405c7fc3424c39a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/8887b12794f10c37b18a90270af99b390eedf7ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/8e8f2c9f42d2f9812f2d788a1f3fb50586e4db15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/90a353ca2f8a5ef681554666074daac2f708f308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/98cd34d7c4d5d313361562a1550deb15b6c930b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/99f969c081f917a154ead25a513a1dd30211f26b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/9d1280de8dd4d771cecdc0981226f5f043c28345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a14420df179c62d7bb2a5a3a398ba5362282daab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a2639a320b274ddf52d89bb50771c249cca7ddfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a3716da37ec3b0bd633614affdc85ab898728097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a3df3ceef75438d8c82b4923045aa0936875602c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a4d510ebc465f74e33d3bed9d591269e663b94b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/a6209afe0d2616c59ac862ea1e25263778c6cd62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/aa4f7b095f3d05dda01689f3130597f5507e9afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/aaa3c3d8d85a7cb55b88f9d378d1d36d68c593c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/ac41b0c80eb6c15fa7d67b5b8dbe8b4edf18a919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/ae79d9eeb159a5d45ebc807424d0e140bfffb1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/b798dcac9a501e7a84b09397fe4d90122d02517c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/bc0475b84f0d19d5dd0e2b60a82e3717d454b479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/be136b4055a52eecd60309388c1280ec4e4af6b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/be6b4114ff094fa74dad20c836a7543a2fa7d7a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/c22194f954f44cd88d4ef0f44e49fbe67484b5d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/c3e19c72603280232c5f865e6f11b3eb10527b44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/cab83f5ea1f9425dcb78043fbdcdda875153e971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/cd39b502bb086068325d32ad7ca8e54ea0608125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/cfcd3db60a3511a87b51f1eeed46c04720333639.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/d1666b9ac4c04dbcb8ad4b8a04d68b36977fa610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/d242b8cc0f8e16735e75f24ee6dfab8b6fbcaac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/d5f01b5a9843603f1a0eb338bab4c34f65816837.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/db594525fb03a0ebf713a2c21257d9104cd9f865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/e2a50f2b2131546246e168f0e582c03184ad4b33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/e31d08d206f831c8bbaf15b5002def2f84d6b4c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/e4a28f975bdac3997c3ef6f190170836da27eea1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/e5d55700c79bc9a6d3c12c36ea143237765e5a6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/ef6dd2f3d74dcbdb0719d4cc0e0818a34de61b96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/efd9e6c8817ee5946757a37650ab91c4b3158bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/f17dd5142721b36624709a32489c9ea765e859c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/f1c99cbbf8fb80c5ff9337d28f635c80e8a5c948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/f46cff5a0520c2b27e5af57dc704c1ef014051a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/f5b4277918d4bcdd6af8db8704d9644b3edfcf83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/fa8fec4390cd78680b53065455de2e6d16df89ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/fae058736308e750763e771404c5651004d4bc14.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-09 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-09/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-09/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2017-13.yaml b/configs/cccc/mixer/CC-MAIN-2017-13.yaml new file mode 100644 index 00000000..adb00ce1 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2017-13.yaml @@ -0,0 +1,774 @@ + +streams: + - name: cccc-CC-MAIN-2017-13 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/031540f05aaf8b728b97ccc6bbe1df76b62d6094.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/0bef1ab7946f6100359bc5f947682383aa803e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/0e046993812030bbac27ca3a9e392dd806497ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/118718af885c17b26bd0a4bee318a97deec518c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/155902df16e8d7c0f50466b6d643102984baf4bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/31624fad2c942d8079b8a419045d396628838b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/6f74461800a87dc510501fc937487ff4a7646712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/6fc80f83ebcd38953a46df6ffba8db2da0a3e516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/95283baf771ea1ca989e30085469c5ef9ed0c8b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186353.38/warc/f14593547d35d33d968f6f57e0666f5e12b7a435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/3887d7d3956becb21eabffe6281d14839a4cae45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/5bfd303a883152645578dd7e37f499094b446746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/720b857ce87ab49ae21095704d3e90aa16bae2fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/8d097370a5135c4190b23ab611c36b0c23129c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/afd931745c8b731f156a6cf2a642de95559b23e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186530.52/warc/c09f2208be7ccbd3018d050a3dd18e97028650fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186608.9/warc/b10127aa44fdf41f4c3be3652fd5258751f7782f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186608.9/warc/d5a6eece8d284c08d8b68d4a7610284970cb4957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186608.9/warc/d91ced3c37ca398ac6de364bcd79e71d3543af61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186608.9/warc/e31996816943eaeaeea8719e26241fc08a6dd1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186608.9/warc/ebca7871fb1aeaae2cc365beefd77a8ebb9f66c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/377324d0d81a24c6111ae958590a0e7af9b17c1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/6f1e015ab4fe1b3b2f06010e23e98ee961d6268c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/946b64cb90719ecb48413b91085897fa4c0d3a9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/bf758bf784060c64eb4f5fe492e8766577e6705f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/ef8b1933d189a3783f5ce910dc89837a0485986d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186774.43/warc/f725520c1fea1f54ebe7e870000f0ae3b58f6a0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/112a4894f7aeb7de547bb8d5e26d79e6f90707ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/1b702b5f1b0889f0b43d6209ad6f7e904c74b9e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/2195bb9a893befabecd96ed7d7096dbf530a02a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/3f9e2aceeb8e8dd0e9008cfdc6f109b5f73ae5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/5efc2e06c7f0c6114264db1f1fc9f0c3e3ae22cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186780.20/warc/674a4968059cea33bfe92e6205f1fa744c5adf93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186841.66/warc/41bd5b1f476ead01bd1c1e4ed23f936c0f5edb39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186841.66/warc/481893980173c69679528093f80c3aab85ea7761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186841.66/warc/55d052dbd2fb33c856e4291dabb86b9ab60aaa4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186841.66/warc/b7921840a99f83e786ddc2f8a9d714da2d2e3904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186841.66/warc/f8aaf01a59b7dd2cba2bd5faea0cebfb57572e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/2c6ce289b1861f1659f5ae48f99e0d5414450011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/65aa742fe0148e04b521da28920b1805dd0b5058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/778541e6f483d6b94377cff8126a1434d1bd9303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/cd5363ae8d6db02e0f26ebed83a5088c30d0b86a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/dc641ac57e64c80c6ea2cc2522075ee2864d7f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186891.75/warc/ee3d404d96e5970a0886e91d8bb3c4b36040f18e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/20a22e987e7edaf3de79365e1444e14664cd0807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/665bd94432c6fbb97193e3f29c98e97e21705e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/b0c9507ab6695cdba516a3bd01c64926e78f6597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/b6756c04cc59b4d591eed72725b2296558c029a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/f784337fdc776f2a45fd8f4281fa226b36f34a5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218186895.51/warc/fb8a48f149952a16168bc72656d703c6c1406273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187113.46/warc/6a35230536d00e67d491777405d25c86b67c3b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187113.46/warc/6b25277aeb05630a028736d062eba61556c58c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187113.46/warc/7b827b65292b3fe149bf6a05fdb564e862eb0906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187113.46/warc/cd5b428493319263d99adefcf374dbaba24321cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187113.46/warc/d86860c1d7fd6783f97b7fcc093bcc1f566a5027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/37e4033e73a97fd2224bd50aef419539597aa339.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/3cb91d2aee2ca6cd5992e6554e17812acc75b3aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/449d075e262ad1d86509ead5d9f24355a135278f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/5c6c4b3dbd7538b9ab66eb9cad84cee213f3f2c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/74a56c27872679774eb0079284c4521e8e787631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187144.60/warc/aaff03f3e6906e8e27e65344da55d412a8320856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/1da6833db617a986774b01cd7f6c61a536069f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/31c4258ee44eecbd9280be7d5f75811a8278b20c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/679d3cbb0b23cdf40837a2a9ad958edbb7c49049.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/8c95132fb708ebd1880d7927d13ea465c187893d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/de30597ce83c8fce490fb3a551dc9f507c1862b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187193.40/warc/febcf396e189a71644bffea539479081c8cb85f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187206.64/warc/1fd30be167d89253de8a1009cbd0fc550b0862ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187206.64/warc/647478d8f1da0c3b7f08736a029a01ad1c19b56c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187206.64/warc/6ed6da1adc15f369a59c7f5fa93e2cf137ccccce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187206.64/warc/85ac3d5b08336881d636ee1eb83e5b43b9ac6182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187206.64/warc/9c3e6fa0b17e89d0ee6eedf138f3f6cd8ece067f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/12e65806529ba4968dfa0687eb52a348574d59ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/27c96fe11fd54323b98e1b94f91241b758069b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/4a48f86b07d2091fa9e9f7954624ad43cc7018cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/9935a1f31f16d2c78d51db62075f82954749dab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/bf7de6fd4f93022cf55d86d3812a6bec10f2a5ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187225.79/warc/da2ebf9148481779529dcd7fb127ab8c77e96c10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/030e2e4c2b642d8bfb4cf43e828b3b2283992b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/77ecef05e65eb202aae561b1f7e1b8f4aeffa38d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/bd1011d090448442ee9f0c9b659e19f110be3d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/c0d2b0f24de504fad837fb9e46b6087d18d05824.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/dff068e3a45af93281e33389263f9c7ebb99117d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187227.84/warc/fc0868041378742db7bf522cef76c92b139e0cad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187519.8/warc/43b1ae2be1d355b6414488e4ae8341abec7b07b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187519.8/warc/8586579e5866d1426f12b4f97ab480ad6541c130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187519.8/warc/ab74234149147553ddbabbebc5734b7179b0c190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187519.8/warc/b183b0e42e88e702f3dc6955a7d979759a3d01a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187519.8/warc/ea3fe24976c78b677c6a079c0bd205a8a86199e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/1e99d4fb824545c6820de72109d110bfc8101d93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/5164a21cc36bb8e0f185c61eba59dc0d646b202f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/768631222eb3ec9839e386e00ae73a63ac096a10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/8209ed1e2dec856c810340a46afa915f187be76d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/8e696082b6fe981d3da4f0bb223f73cd707f56d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187690.11/warc/e569555b3c8ab871a4d482dc818827b5c4c8a4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/0acd4874a5f6e76313afb8b061e2d9ae2cb6a5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/293b128d15f16f2d54be7639b4c6e6358c5f27ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/2e761e53e36a20a556188ea68173525720951d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/31868ee0fbfeb0af286076ac3b631c5661e38e6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/830121574b8ebf6bec5b110331a07a7ba2bca5cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187717.19/warc/891e73f4efc7f8437b94f6f314b92d5f00043e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187744.59/warc/804d8e2e4c3050812329acbc1116c7c34185a0aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187744.59/warc/88f7565019c399215a76fce277afa23f5c616d54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187744.59/warc/a8fda961ae67bbeba210898035b10d36fc022367.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187744.59/warc/acfbfbcbc7f61fd371d8dffbe40b11e5ffe72220.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187744.59/warc/f1afb2b8a20c30d4807be8029c0564c60664088c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/4c049a55ba22953379bab835087518429d610fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/5152b51feed5631839ab31e4eda984163b296a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/9a7dff0fcbd8495fd94ff0b6e8f0e230f217f32d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/c2b0b4839f003152bd0b6f9b51080cfecc4c573e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/e21e9dae8c181360e6ff1ec1c431fdf1abee758a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187792.74/warc/f824f08aff43f6959ca0c6a35ba481a0a56a20fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/66472b6c471d7040226af93e4a30086fd9127cc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/8977b557afce521196b68fe3c49dbb542833be60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/ddff6b0dc7895ece8a58ea283cd1729112096e5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/fc9bfcef6841121f2a984a9eebad0e3fbec6982a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/fef9153d6682d5dffd8506f79156234904d3db2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218187945.85/warc/ff23b46103377c43128f6e0cda3c5a8eb60aa1e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/2fdd676d0c15382ef9a7d48dcad905e8dbc439a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/3a8f3445a3092342107cfd7e98054ca42b20971e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/7124a9b186258cf62edb5c13d40a96236dbb6b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/8231db430ba99e58a836fc91f149d854ae5c094b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/915b4c57dcfbaf5f4708c368e5979977dc3f3e48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188132.48/warc/bea4793a914e79e7c1e713422601aff881511b75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/1c8a0ae640a32b2064d72c1ee7b0bc9470cde94f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/307100dd9047d78ac5f5a43e8c3865e2f1a551eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/a64250ab97573892199cabe46946b5d25181b36b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/d96c3d6098d3a44e49bf1eadcc603eed93b4957b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/d9d8d763676393c3a41d9d0f360c887a4236a4b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188213.41/warc/ebdf721f9687c5365cf87f06706c8ff8f70a95ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188550.58/warc/1672115f8efb3b9faac361ff21197f1b6e9727d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188550.58/warc/8477e4185cfe04e1c97189f2fb7fa6670fb45943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188550.58/warc/92f3f215e1aabd038efb25c5e4d3c38ed0444a2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188550.58/warc/957af685935d8ee0d6f33d7bac54f258cfeb72f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188550.58/warc/ff1ccca8e390acf7dc450cc26adb25f18df579c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/0ae3c49f3b5f086bcf6ab45cd5186ebf9101985a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/1ef2b4aad234ea1829c9368889367478d1225d38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/3e741b12cfc5284c190b5d793d7fcd3bd32a5072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/6b4e9cece55962a476fada5243e2fedd1d02565f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/7ff33c79f88df8414e19090362e19f25faa093aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188553.49/warc/b3a9c3f7bd7bec80d658f15e7033938cf107d954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/1f96c0bb5ea60896a8a409b43d394aeeafeb53c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/4d09655eb6a7eee538731c9025be6e284707f97b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/4e8748f4c50971f2d3ddbd0ef5af28d02bb2710c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/934bb1df34074d94530b2b7b0331758d1d504917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/baeb3b16ca3d59b15ab73670b8eaecebf1676e43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188623.98/warc/c662ad5f611e5a3c3c60ab8b3a6a99bccca8b589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188717.24/warc/162dca2b9236998deccaffe28c8675ba4b6f5912.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188717.24/warc/9c29d33f9a736e94e4961411bcb92d4da1588ded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188717.24/warc/9d93a7611df89c48262976a307b9b29ec744eef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188717.24/warc/cc7badc98de2ea0f4b6238eb8f7c0d4cd7948b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188717.24/warc/ece6129e89f6fa4ca2578a3e226eba1569efb5c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/11f41a0f93336201ee6782c592a127d0eeb27018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/38ec71d79923f8cbb69b0a46957210e436063564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/4986989468b3c52b9f6e187ce335d4e0c67c10ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/555629ea79f2ac90974156b962522e92e750e184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/aaaa47733ae4eae5ad0410d6b7529cc22a062df9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188773.10/warc/bb191bc1377af343a735658712e35ac90f1f1aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/140ae2f47aaaa8f870947fd8d55c65629436e5c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/35ade474aadb8adb37b05f498569b0350ddca366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/862272e201ff9bd1e26021dd4181717d3d248693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/8e947389398e256957f88b26621e5f0eddba1ce1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/8ffc7e8620af6e9bebc5f28a1d81032450af87d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188785.81/warc/9669c034bc37f371738fd29c30f312efd851c86d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188824.36/warc/00bb049cba1f32167bba3aa6092814929848be4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188824.36/warc/3c627b54335d7fa2a1fa98b3f8dc01dc2f89332d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188824.36/warc/959cc49ad665b17b081199b88c03d015a88b360d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188824.36/warc/a17a9f7a2f2bb47db07396e9f50676a71a18c201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188824.36/warc/f5dcc3978e9bf641c2df4325222cac26ed77a9ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/353d8d4925cf19d46327749033f8e012c2a9b5e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/6a0381e2a8959772bd656b9fda87c2f85041fb98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/6a7396f9eea87ac3cc088ba8aeddc4b1f239b592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/b4c0737ec6a42e90008bb7076f0cfec941d8778f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/dde0e9d8969209d164a55dcacdcfce356613bb09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188891.62/warc/fc4eb985c48a629a3230504ad824f7171426d1a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/091e52477a8c5e6df7576a8e470cf42b45535627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/1212bc9dfc5155a28179d89af562492b7fcb10fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/1e8ecd61656ef6a1ce8583626502577ddef5dc01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/2038b81ba8f5b4a96199c9a265552a9f6aff6769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/40def3b07dfc85f1d4ac825bbea56afb72a1613d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188914.50/warc/ac22a3dd182704e5d69b3c60ab7211d0b0643963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188924.7/warc/15fb70b46faffc7b44429657467f1fe8916e6501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188924.7/warc/713aa5f9dd3c61df180b2ce2d2e27c09b66627cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188924.7/warc/9a7264350989ca8b5d5cbc17aa46a8df545cc3be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188924.7/warc/c51594c0ad7386591dc34dc68be70da7b645556c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188924.7/warc/fd3a0d0589fae6bf4ae289c24e5766f01adecebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/0d38f3403b999e2c3684eb6aa08c6a64dbe4d55d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/186f2a79c5aa978ea178afc3ff0a037b195bc163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/49e92f3ad55818fcd4f73a2af8dcaacde396904f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/6a1848d6f8b4bc5f3a9dfc9a1afbb72486aec391.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/a6099fb68647a6becfb1eba48a18d398023b6fb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188926.39/warc/fc5318781a805c5cbccdcc61df2684374a03a9b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/2c22c9092de3ffd77105d0f869123f5003d70614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/7af416ad24b8bc6e92164f23e35e89f357acaf1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/7dfd17b5f84fba77dec743d823c56f01969ae7d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/8a1e618efb5761a41f1a785a8d789d8dd27f5c44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/df5d63a455bb3a5337e5c2d3977a8af0cbcd657b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218188962.14/warc/f158c5bbff46b97a5d71a0da5870f47c9d11ad58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189031.88/warc/20d70cecc06989f405eb22cb8aaf0bd79fbc39ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189031.88/warc/763e6b07be08e557272040ba8e2ac7e89afe740c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189031.88/warc/78bbd11027fe46b086342e03aad2349e4667934b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189031.88/warc/83309b18b825a38e2b03b04b7ecf03d18e3ddfb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189031.88/warc/a3959614ecaa1793ef66cb1298e35200d23b9e81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/4631e980068c35d6999e1e0b9eaeafab92d270b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/5d5bb14550d6acc66314cc821ca43b560bd64376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/8aae8f5621268196ab3f6d78dce7e3b35b13f660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/af90ab2517749146c67448c97134395cc235fcf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/e65fc0cdbdf2d36a881e57dcccd543702d54499e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189032.76/warc/e94b533d917c2930641783ea952f351092340072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/2a73581c67adb0ebf427957b03ee6b1f5d1d2a87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/2c82872ea31f43822661ac371dce460e6713aa5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/4717a68d5bdd15ac22c5b6a1155009b0f36f0ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/acb14d205727fa4bf4892637970757ed5c9b5648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/c52e343764efe1dbb7f15ca2383d099c9ecda063.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189083.86/warc/d455d8de44fe328083af24cd78a1245fce6630d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189088.29/warc/2cecc10211cb97602bb63c8f00c90dbd054382fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189088.29/warc/43f580d87e9882edc3fa02a2fb3f306a52dd5ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189088.29/warc/55c7dfc56f9b666cd8355d1be291e19750aaf2a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189088.29/warc/c248cef7ae141e54291118d6f866270eaf4162cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189088.29/warc/caeac8b5312f1c2df45515a3a3c0125a332e95e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/211f056d1ff1817e7484f7dcb40c12569e97f27b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/3279eb7ceee21775dc9fa9a54c1e4d801c91ee33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/333c8373f6a56612a0c8bb9b287728edd4db6c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/660ae1f3954ca0bacd421224a4d62cc055369379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/b13f95a2b74b8fd5fa9e4be1575231ee284a6a69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189090.69/warc/e17964216a1f41262ca59e62a051437d5cdb4e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/0de35c15e14370459127fbe372c677f8564c7e24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/166362abc079b2162e77da4f25e872b2a3cffb88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/8bdefe9349bcddfe8e2d6c61c8a8b4570c0a7347.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/bb908359650a76e3f5a996f07d15a4a79304b692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/d55f17ab6eea09a75314a550d785f04780053e2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189092.35/warc/e330372642fed2d0bd63a7c7918dd97452314b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/433563c5ff6b4c77476dc26e6fa94134a4e0cbb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/63a2d664ff2633afeba6c241b5a5c0283f67c861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/8967d108fd9eb27ba82414eabcd36b1a7dab8b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/9a5aab0c322a19a5ae20e467a90510f90e6f1a51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/e2930e4b0f2978e04b96c306c429963fa691e941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189127.26/warc/f66674c98050958c473cd2dc1c5549a96baf592a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/16abc540e9ec2033f12e29d4f9457bd7750b9f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/1d4e944ddc0acd7a59b14e02abf2d158fc356d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/2d3c3bee6bf6d6bf249bda4dc2b9d5444f796bd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/2f0ed187f48a04f954ed28ec61bd3cd09fa59439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/84ccca6af6690f6184e0fe4da514cced22ed6f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189130.57/warc/dc665f2f2ec76c9fcb0cd4f0533a62d64d57e8ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189198.71/warc/033287f5ae8b5ad05947f579e8463e1160986d9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189198.71/warc/045d6aaa4c87c40f53ea96dc547e77aa6eb19d00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189198.71/warc/30577079f7c5aa6078b35a302232428f6bbef248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189198.71/warc/a26052ce867a720588f6f998caeb37e496439a3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189198.71/warc/fb611a00ec8f67bfd4ccd4a07771000d0e4e20b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/330416fb463056d173e5a3d42270e5c216a3962c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/479306f74d526a7f4c6e220db0a439c35b22424a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/683caeb96dbd9c22361ee14106fa9e9f6f692822.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/a43813d78368bf8c8a84e9743a39cabef044cba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/b9e043afe1b2d4997dbc5aa2a057d52c5c91bf0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189214.2/warc/d3c2c25c91023e55a1ad61225bbafdf925151611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/20d3acad869ca17e735ab8bdb85cfbd2b19ce8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/27fd93cfd0c05936dc0a44f5cb16401c6f722cd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/6c51184776a5629e2d8faf501f256393af0b706a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/b26c21cfd5be9871b6b340398b5f10e938f6076b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/f234f957c83075c8a5d2ac2282c5bf1738ecdf0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189239.16/warc/fd2abd0bc6e011b6553b67826bb6f87b57bb8a07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189242.54/warc/3562e9bb04f9d9ef2ca5554b00fd8547c2c66367.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189242.54/warc/cbf5f0d8972e9c8c81fd6ee6e0c37d09cb069fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189242.54/warc/d9289a1063ef5d3f9adec94bdbb70d75764e5018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189242.54/warc/dcaa400db07d1805d4b30e029a8ff79cc48b4f93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189242.54/warc/dd8b903e07e8ef4f62235e2c4d5f8db4c24ae09d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/18c2fb3f93b8b4e6b60f3260cf2d067ae7308205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/3355c7480922efd18186d18db4b515cfaa5d0459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/4d35e62bd75c1316d4784d39e85b845b60aa9091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/90824885c1b596bc18b44861edc2709556921b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/950ed084a68b5a912f0ba769896f03aae4ac6f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189244.95/warc/fd98a358490c9c73466dbf6443e1236f2f76c3c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/2d65321b6f011bce6b73e4a16e6885dc58a6cc79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/42c76b0992e6b1300a11c86a65bc1f9f8f42c620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/5dea8d074ccbd725ee720395263a32563eed8d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/c674b1484c8e172ca68f252b4158ffef2c0ec9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/dd0e7caf0b3e214ea485b0cd0407a3ad284c76d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189245.97/warc/f3db4c071d81e37348f78c61efb6e248e5ed58d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189252.15/warc/3fd067d5f885005f22ed38a685d2e6f6cdcda9b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189252.15/warc/7246b7463bdde9bd46183f830faacb4a0278e651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189252.15/warc/b1dd7d13f0d944ba2152c76a10616f5c301e92e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189252.15/warc/cc9a67805e9936cdc3457d720da5fb30688d2009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189252.15/warc/d42a641cb869e9a3113e88231082399529298563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/02840cdc9c86f61a9ec3993c467a4e004c4c6132.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/0c3b3c984cd2bc25cf501accdd11a428ed6a1b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/1fe888dd7d80abc15cd98e159d37bee039af7ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/4341fb892d42c10c983590532f67e31069c6719e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/b87aa5062a6e731652371b8bb2d50305e3ef6169.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189313.82/warc/c667e11ad258d2e57eb712663fdbc07a99e81342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/a0757ed2acc1f0c6e0b94b80dfbcba3b060ddeb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/b4551ea11fb4ef2d82f3a0e908d9a364a7ac256a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/bf65f4762d391088fd457de4548b7d89a7b00f3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/cd1843b90b5c8ff88db52366787c98dd1ad40053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/e31851227469851b0d72c3d4b0ad025de2e33b25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189316.33/warc/f165a2c046967992384b744f68e8fc11062e30d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189377.63/warc/014d53575c7eec807c61af9a65acf3ef1a5ba71b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189377.63/warc/5d19814d87ba4239ab8d3f59428087b622cd7005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189377.63/warc/af90639bfb289bca0cc434ad2ebed78a652962a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189377.63/warc/c8458f78c0fabd841aac372bf14f2e1e119cea61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189377.63/warc/d28b81cae6ca2ef21240b34c337f778fb2fad474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/1e7797f6a4df2c849376604b8c49b4f00c9841ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/7e98d8b1c57c4621fd377505b9ea25135e97cc93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/bd95e110c8992fa4a1c7905397bc018416aa3104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/e6938b26eb8ae6c250df1a37d79d5e19c7a888c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/f2b58fc8bdc51f0da38febfe9814d1fd2c0cfad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189403.13/warc/fe492894bf43f191a2d399d0e75de341507e6f75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/0346382fc4ea5e6ccb6635a7e8e780470137a222.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/3225e22aa1ea383307dd57966e44c484ec44c1c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/5d14f0006725d7f2eefe92fe734f1478e292b22d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/b0d4a72478269b7b4b93fc6c9abedd8481582a69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/b2958877f3576c7ac1af30622da19d67835b684e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189462.47/warc/bdb0dff0d3df6782f33a832d9c19374aae3a6f8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189466.30/warc/1fc51ba15fee95afe3bea2c1f74fade649a62b17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189466.30/warc/50f676948f1e679a00a37e5871bec71cceab36a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189466.30/warc/525531c20f5420f4c43261babb3c4e5bd585f890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189466.30/warc/8ffc2939ba3ef4a3d99afa919d77113d28ee4b84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189466.30/warc/9116fd2d2328172f099986de57365f17958bdba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/0f188ab924416b026dd567481a26660234ab1113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/2101fa71b8ca3e5471b2a2ebbf2f2a1fdad03c43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/4c48629e29b061b041b6148c17c2f96a76437473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/6078222552824ecc0afc34c5ef1424ac3e9af20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/6c2894ec897e0b2c4b0eaf1d097ea42a7bfc18ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189471.55/warc/f810c8b283ed8739bf7380196ec7193a517cd580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/1f8abfbb880d9faf830e9e7de2a879f7c85c07b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/2099b545bff9c419dddfcd476e40f7ca491aedc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/2401980b826beea2d8e6aa2d55afcb2df472cffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/4324b43730a042242485ada8293df2a376c7d995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/608ee5a970954bd2456d27b2bff4e8c01d9bea2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189472.3/warc/f49eec844b06395bed177dee519090536aece909.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189474.87/warc/1524b7dc8968f73338d8290e93ef3eb0bfff8985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189474.87/warc/1ca9834e4f3dbb3ab784d4442b2a84715c1a1bf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189474.87/warc/6ea14bd7f61b6adc3fd5df340d9e4a0733afefc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189474.87/warc/ab4f1bfe5bda110c1d16611d534a86f845126716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189474.87/warc/db645af4672f1dc1cfee9985ae61e572c7cefd43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/77e68888e8c7beba51f130047e761cfb1e0c5989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/89dc0890cfce82fd17599c07f513bd6a34465bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/97528e41950f95bc0cd7b1e5b7f341ee19e9f9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/b890b7ddd9ec318757531a03e0a5f8f1b9fc4f68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/e50546b714c2d3c12515230f9fbcd38bf5b655a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189490.1/warc/e83d98b8eba374d2bcc6337bfb575a9760671a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/10c751065914cc27d914c4bd0b65ee60aa1b1b50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/2e172fffa38972eb1dc85a830d4e4b40eda59b3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/312385cca475bda552ecc5c8dd2eeca4810bde18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/b17b0ebb8477ff8785a36c625e33f29d0832d899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/bca08bd9d60bf4bbf2cd56e7dc423ecc873ee6ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189495.77/warc/ee1edc7101382b74b896ab37898b2eb9d0c34715.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/1c83d994f15689ebf35f23190ab7f7447dbe3d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/3187b9ab7fdc08010b2e266b1178316036d221e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/6e057009a87d832b406948c85aefb660d174cd49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/8c44a04ab061b57e131d9482286dc39ef995cae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/bc9c1fcbf94c13eb382d48e57863d5cc1706019c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189525.23/warc/d0c93c64a93e4e1409a89328f840f67fba3cae91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/42fb587663711a3e10dfaa58e7ae602f69c35bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/677841307a523c156e8e27022ff947998ac7e0a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/786f768fb34ee7802470a29d786e17238f5d2927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/83e03bef37eb43d462aa38f24b12f56277b5f60b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/86fce61c1f0dced3e26ed0a0dad15e868dcaa3a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189534.68/warc/a6ee8c7f4b5035a7a46687d651c4cec294fb0e9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189583.91/warc/02a7e84abe98ad8c87b33380d6c1da7b9e6f26e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189583.91/warc/6166e7c10590d5f8bf6b5665c851e38af7eabff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189583.91/warc/6e3316d736ca28c8b6f689306b99c5f5a65571f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189583.91/warc/a013f453e297d53dab96928ed815e692e82719aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189583.91/warc/e2db8f96655f440d70afe2bebb846d504d61c6fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/5c18ec03ed4b0ad21c30c19f7d5001ed187de535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/63014f2207ccf964d3612cd7747efc5c99b57817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/69fd4798081c2416ba1a9eed436e5ae36f230f9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/96a17ba96382dd9ed0dc06e7741f9571d51bb8b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/b641701ced5557e772b385c8fbcada421a152271.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189589.37/warc/e3b2500849addd26d6a17e156497719675d0ca5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/640330f38f79a093179d264472440e4bff79d33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/7003317d3519dc65cc47cdce1f2fbb86e1e897f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/7129d6ba469cd615ae4162f88bbb1747dfc95c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/8ae57e8d7d7d7fb8022374c3b659c733af0e5ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/d77ce850ad0a5ffe3a5cbfc59d642518e32fe513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189667.42/warc/fe4b4e4ad933fa5bdae992f9411f6b0749c3b2d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189680.78/warc/0aefbd8fb6e941f584b3f2563824b55063b9094e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189680.78/warc/1d4337ca9dbed728344ce58d9cc297564848c83c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189680.78/warc/28672ba9e3b4e35ba994f98fdee4a499d48be73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189680.78/warc/5e647ef20c30f4d3e5947e89da4707b2d026bab5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189680.78/warc/ea4d8d2dbb6936dbdbc9993fa756b4efce4d0571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/2120704ceff338ca5ca888b94269aa0e7087af22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/2139799e7bf1b475cd23ce87e9f2222d7b916f04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/324de28b8170cd6dd53cf6a41f655304f6f454aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/5a2d4bd96d3f245a96e7033c11544fb831384e7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/bf861a7376d6992c7233e2200676357e511623ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.31/warc/e8945ba7d7539cd48febf4789eb2f90f4c2ab8bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/3ffe2c396c5657a93455dd209a6dd460ba26af5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/8c7faadae06be7940b251633ccc35412a2717581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/9fa34a45e46c61b015b6be0b85bc5999b6cb3cdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/abd6b5e76846c0f29fb63cbfa9120884974278af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/ad789e3fd84303a67d7206b87361e1eae9705053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189686.56/warc/b6596334936b65394d2fb6df2b34c9cc50ccd68d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189734.17/warc/01a92dae933afb4abc7c897dfade820d041c78fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189734.17/warc/5e28b3879b0eb139f7b0f4fa9fec956d66ae612c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189734.17/warc/8246a809a7e67ac58c97f065bce28970795dc0b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189734.17/warc/8f0b080ddfc0c207ba8105abdbc15058f64670e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189734.17/warc/d86c886204647d2d9cb393806b345aabeb5b3979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/20584ca90c7bd202a3ccb3298a5d0541dc8120fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/2c140b27b1072a3cb1e15235b7a1c4c88980fe83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/5ac3afc8fe71d7db7463d18af21a870fe873e9f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/b385eb8c8c0351a2be478871f775f4a665914307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/e4ff7fa9e7900084f95f9a5a2ac4403f2fe232e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189771.94/warc/ebc839756ab7703ae68070c9677acb582a9fb1f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/3d6fc1a6c230d11f7ed9f154d946ed5e642f1d51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/64eb163c65ffb992a70984a92e5dabe6b2940e29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/a8029d5ed482483799e1fe3603d174944995e9b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/c2e6e6e9ceecebf4838c4625cc663c34b199f464.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/c35787a44f3d4e814b9378e0da0c72ce6f7e2fe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.18/warc/e2f413bc62e20fabe9bbede73e15080ef36df5ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.72/warc/061a6c5a8b0a771945f239409a71e8c816897fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.72/warc/1354f7ca9e7523616066fcd94493d0aaeab9d5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.72/warc/16b5ab0d98bb929e5dd846aeee7ce162f58c7da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.72/warc/d8a9a1daf2aef75281addc1c314b05be9d7401d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189802.72/warc/fb249bc95a54c31b60a085ca218d815e9523b787.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/2d0ab37ef8cef100fa695889e10f50e1fe9f2b7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/7121dfe1c44d7e63b5ea83917aef01de3e7c36c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/8f4a476573501eadfaa2b168fdfeaa863e62c6cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/acaccf111d1639ec28f88a22ea10a4f0b48c421d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/e7dc7e295003e54ce5cc06aa9e65282d731b2a6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189884.21/warc/f88da03bd2f4a2e0772adc1306ba70c1d3c5ee56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/17d96e9d12cb4975b3081994405687dff6ecd0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/2c34a81316ead1a0726cc3be805bdf051853834f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/2c8627464fe2e6e82deee3c2f2510157bc655246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/4fc60e58ff0ea1874aa1975b7d4fb00076c41faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/82041f3aecfec7c65526d23fae0bb9331a34ae47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218189903.83/warc/8a2f913eef318abb5f4b0a97bdecfc11149a561c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.25/warc/0c5bc144b76e2a87c10a9d1cc1971c905f6f4605.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.25/warc/1f3861210c27c4ebaded021eeba153a263cfe338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.25/warc/291acb746cf2e459411a37867cd7e745f2edf3b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.25/warc/2c03317cd8339cf47cb2410a9f433fcad6d82856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.25/warc/48f169e0e696d1b9ec0010d6ab29b604e8f7b13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/55021e7677ebad6c5dc5e996697a1ee6857b4c7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/5cf685ea796ea5763e5ce34fbbd0cee189326af8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/adb83067e911a5efcf956da4dad95c4b2604d856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/c2478e28cfc432664e6929cb5a3d2f0eafb5044f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/fc03529d7ac57c9543bccb2e7d344922e1e0d675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190134.67/warc/fcf197bc6dc0ced86081422af74dbfe6e2a76491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/0dfa3478a8a684025f9751b1358ffd599a1f6cf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/1b2296083fce746095af5c2536584f96d7bd08f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/370a8b5355a24123a05464ea5c244ed8312cefdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/6bac3c8c6d6181776c62baccbbcebffa357487a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/aa109d11a59a9112a86f65e4eeea2b789b8593ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190181.34/warc/e172992ae005f6f67bf03811512ced0f9e73a7fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190183.80/warc/5a06afba6d4d6a91fe267f10708a68907df19428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190183.80/warc/5d03726794439c466fb498bcd38eb8fa2b3c961d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190183.80/warc/75339a5835ade8f8ebcc9f8a7ffe3ca4dd672a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190183.80/warc/aed4e104721fe72bff1f343032882896a6a0040d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190183.80/warc/f0666819ed196533dbe39655047b94d192ec2d23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/03cba226155aad407101190a79c3d61023c5581e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/0411b6f4ae1d89edc1e6c4d401550aa4ace668aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/40dae6b51586c3ef2ce465cd73b868e12b647d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/62bd77783cab146047ea55db9f7ad4d788084f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/687d7ccdd083aba15165878fd5d28a3ff8367550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190234.0/warc/be6daba86aa0d341c4f09979db4eeeb6514270c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/288f15f3542e6386c8896ab3f4527999a7744aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/65bf1ff7a3558ce3f56b8bc8243990b3dbf0cdc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/88fd04ce65a411f46d354cd7bc852ca8be819046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/a5853bb041cf8dc3f3fa31c5224acb8cfad66c4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/b35f259d449e862246f20ab34a6ddfed94928342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190236.99/warc/e556bd05418d44bbfc77d0413421c1ffa2ea0eb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/02cf03416e6ef7ab3c0942a2cdc5a5bdf72ed9de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/204da00017f3aca54cefcebc4b5c0d076f112a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/757b24901f02e78b95a3c7369dee14badb044c8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/d931dbf71b6cb2e098e40698d25d5459105de990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/e116b3520117782a55e876655da71778998ce866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.4/warc/ffc2cbf4a7e0ecaee51c40e084d6a3f402c87c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/4d2cb21613a4488c8d5af0e57045cdae35ad0a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/7557e30121f8af8db6adc71ddc15d6997edc2634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/9929c71c398bac3f168fbb07cb9fed18299dc5ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/9c05646ca885ff3a4e2c7c0f7e75c2339c4aaadd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/a44139dad92f0d43c51601c89c1a19e0b81f50fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190295.65/warc/c11f7ea327907a2e3a3d50912f80ddb4df41942c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190753.92/warc/0f6437ef0a7beab866ad58fde2283f4138ad7f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190753.92/warc/5bf86ead954f6457aad0a4e4697041f26121688f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190753.92/warc/cc92c230b89c6aa4a07afa1d0d82b48deaea9924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190753.92/warc/fdf90e31dde4f3f460d6733f0a8aeb293f960c71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190753.92/warc/fea6ef8eba5b329520c0c168df18a183d07b3715.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/050e0cae1676aa0478da3c57a3761aeb24710212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/1560cb02552ebe34f6c427e52e389203b958259e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/401231bf63e109dc14907aa120c71d5aa4b9c598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/4c639a499649fd43f4afdaea8d374c6062f10479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/6400d9d8ae07d4e58100aea4d592fa2dbc9ebab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218190754.6/warc/cc8bd9e847def2575e9614202974961fd2b03d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/498427bd872939817c04265758e626ae1451523d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/c3fbdef3f40991e1e63a92dcd50fccf5a00d6635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/c602e504d78059b02a8e8a5ceb0a3b2f948783f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/e29a34e25f75050c5a1a8db5c81ac3e5203d75a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/e59a1a67e9a22aa53bc202c1ae15d2347108e556.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191353.5/warc/f3a1d2902343cf65af86277cef208d40efdbbfa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191396.90/warc/18df5f14a3e385c003a8e29b2c08ed9f1cca952e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191396.90/warc/2436a3d69e8750ad524276d530242c71cf4837de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191396.90/warc/407547dad1dd838c8fc648153337f38c5626e2e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191396.90/warc/5739ee782b86592ae7d9ad5cc18e3ca767df5b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191396.90/warc/f2e08d5d18761d3c5a6d0892d8456195aa104d6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/0035468090db6ba8b54fb350a78c2be1fe9ba098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/2554c54a878bb38a00e042ce9414454c07c6b21d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/2a9b7d8fa01e156b51cd24bf45e5cf93696bcdbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/31a5366d9d17f08d9b1e77ae8b75946c6e73eeae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/3ee3927f0179341e6c73b8f0b7bb10968fedb32c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191405.12/warc/5078621196798ae806de9c751e33164f2d5f8343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/31e72dcfa07f25eb5da9ae7e0165d2ba737e298a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/3b9fbadd0c121c36be48c5fd08384727d6916268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/4267d9117ef4b9a2d2306ab154db8b097bda400d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/45946d7b67cf538d1f764ac0c17c15c07101165d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/4d65be331f4f7d72fec76ea2f6ac0059b826272f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191444.45/warc/90e673c438590e6198d37f4c10121fbb76e0555e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191984.96/warc/10d89ea8607fcdaa33f1b393e8c0aac0b5228a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191984.96/warc/7bf42418ed70704df3ee2f972f70d4a7f4809b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191984.96/warc/96ce049b62720e3e252097c6078c0c3109b2016c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191984.96/warc/97e73e2e03e2fd1319616889770e37e4ecb29cff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191984.96/warc/ae9cc64d622119df1615c919d1453c9fd682f517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/17426ee86c62bcbffe2093a2a12f8c9197aca142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/1a9925ceb9ea5f1c8b095a4b011d921d0913909a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/a24f4401d1477c47e6fc4e2e3d02b4dd67bd4555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/ae81a123a7aa691c9f8f9e2768ed456a4cee7f13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/c3d0a90976d29e439298e0444f43e396f5bc5db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218191986.44/warc/c92794ae4074a061e8659a843913e1333c76752f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/3ed2da9f9ff98157f49acb782013da8205893ec4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/4787eab1fb052da0dfe53c31530bcfeaf41e0e91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/5dfd6630a92bdf41f0b32bbb9b185fe3d810018e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/75c17e838675dbd4e967397ff1a102ac4bd0b15b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/7a8a9973bf03a62768b1be2f956c35a91f4185d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193284.93/warc/c0b78a125d038f2b96931081f6db1067f97d1b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193288.61/warc/1a4af655ea263ba99fac856dc9949ac4f5a0b3f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193288.61/warc/26e562723578708dcdc4e6f533f6bd32ed297967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193288.61/warc/6bc5c04f27f43f39bf56790101029d1b7c2ddcb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193288.61/warc/d3af9c23ee6bcd5f34e8b13b4d7b95abdb656044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193288.61/warc/d503843ad84aa024cf617f1e8e5be32925b0ab0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/2b1c434382e5da365de48f30a69284523154517c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/3df93b3ce2cacb66ba756e7cc6046b65e4bf7328.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/78749689e03ac6353b5a0604cd446f7d57cd0a5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/8224c0bc2ca122418cf053d8970f5c4de1b09a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/9d823f2fef70fd09dc3d144850aad712d074e190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218193716.70/warc/f597f9602f47bcaa274cd6bb4352e04bbb49ffaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/0e732b3062dc4f0706cd2199c152e3ae2ab11ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/412e9f0ff71bc5ed950b315ee2cfb87cbb7463a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/6d452174220d044f498fa8d65263988a79fe2281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/9d43d80ac6a0d54484c17ce3fd0e2ac47d9ad1f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/cd45dc847b50414b9be34df8421383915399f9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194600.27/warc/fd6be345fb921b24193ecf7a0d317dd2acd24381.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194601.22/warc/1ecdd8021445915d6710a148418cee870dd5e987.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194601.22/warc/8eba79e3651790e0b8941e00764caa308667d465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194601.22/warc/a754ca0953d9d11969f5d2fa593087f5b5814ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194601.22/warc/c45bc84b0c576099433d05b2c9aef5c8348c5e43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218194601.22/warc/ffd245d7d671866be7668c3bf95ae474c30f72e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/23da4f590c83336f4a9a5d130b456a0a367e325b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/55159107402c70d3de532d4bded7a9e7d7afdd90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/5e0568114ccbbfd344aa404c4356a2614fdcbef5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/855d1715e6f06ed6fdee50be86cd39ef9be141f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/a69d62d4b8334596269b445e3d9f8cfb05e012d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218195419.89/warc/d94a0eb3a5396f844a22fa765c3083bf2af5a38e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/43cdd30fb1193a2af550a72709985f2c38069bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/6eb01176f43ce00e0731e297e93c038f1b853476.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/7757397d2f1788ae1e0455f0d72397dcabb1093c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/78e19c7eeaaebf78b725b5c03206268d6cd5dc49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/f130995d5a4f2b73de69de3b0d989776d888c2b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218199514.53/warc/fe576932007d06494e38882ff9d2ad9ca0b9b12d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203515.32/warc/1dd81367687d2e64286376470d318ae2895e89b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203515.32/warc/6b4380bf05d109c7d8d5e660122dea682e5c01c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203515.32/warc/bdd1a64915bcdb833c64ff4de6cdace8cc147c7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203515.32/warc/c43aef0994a0f9d98d2d1742e5de682b6ca108a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203515.32/warc/ff61db11e2969cd05bcbaabc68fb939fc7d59faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/37ca2c1266af2d8527877c191bb6bbec2fc5a8aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/3f7e5e91246000f54a907fa0aaf46e39ffde415e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/4b8f1127b12d4168ea6ef52ccd28749e25f83221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/ac7bd9176091c64f62028454d39ddec8912efbbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/c118b2f9e2fd137e2abc29112c3c0adf278b090b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218203536.73/warc/e86754926b94c8df2356d9f8c4882688b790a6e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/7ff92832f4c675c653750f224395b23892243025.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/8145e534ec86cbbdf033c163cab55ee98dc30a80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/8aff637b3b7b5e3ff5ff6c127a0f78c9704afb0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/b107113aba0919ee39f16f13f42edd02949e73da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/caa83b69d28cd50de8e8cbd3cad2c510ee35bafb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1490218205046.28/warc/def8302f2bf9dce59ded2c2103554120285623ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1b48d642e84311c9ae996c942e7248ba59ff81d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/1c49b321d9177ad574160e1d9a998ac8b772baee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/211f8cbbf4b5ab3a9183099da15f56ab21d55954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/27deef0992f9cb9364bf7ff3c6e4912cce1dfb9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/308a58683e429b73134653172ea8c7ff74c5087f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/328ba47eb3226f38cdd6ff23ee109cfc027af7be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/363eccad88fa369d6f47a2b246ac9ac0bb7d95fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/39cbc7785eb5e2cdcbabbf5991c99d6cf7f9eaa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/3abc16aaf0bf0229ce8794e01046f4a8bc3c18a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/3bc74c72a20ee70ca47cf50c64b76433564f880a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/41003796046195857c472147370fc6852cae5dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/417ae479976d15dac5299348ecb0748ac97dc5f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/473e22284cbcc3c48760c90d6ac1566f236998a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/49c164fa570079ae0e33d68defdcbc2575416c9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/4a6ecf4862adb9dfe8d32528d6b78757cb7c8354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/4f4025060aee2ecbc1bf4fbb2e43186d15c04dd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/505c88c3300e38c5e586c2210397c94d352f46b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/5183dfe7d69ddd742d4f91259d6aa4e4ab510f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/52b65e4d8992f5defa6c2c5610d2c4a5bfff85a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/53f3b6177aa24b2c7c38054c9583ef5d62381bf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/57b9adb087f624c42b3d6d38463c5d7bee8c1fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/5baaefa9ece94842c23853f2bd35148f2eee237e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/5c127fb51b1fca5e6e6129b2139a230a52871a52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/61762a2d3985c3a19ecbf34a28bc2becb1c0e1bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/6176ad8e6d15d0f3adff155e7e1a40b15eb34d9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/62c65ee5dffe981d827a607b0e616bff680c4672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/64852b8e7fa0926768a2c2623bf5ae738787cb78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/65007eedb1d9bfa6a3f76ab840635ed70a7bf1b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/697925ac0dcdf10968a4a7546e361c74e7b194ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/69e7f33cc07710dba8718319599a5c5c9bb12946.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/6b420e8943abcfac6697470eaef1e842f20acd07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/6dabe1ddbb4d5275465cc647d63247baa560acee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/6fbe76e47f2fd8604726e1ec9f82824f1acfb67c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/73cf842e53293a0043987be55abd089c391fc6ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/74c0861d94ada315adf238c60b5c52a16a893644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/78cfcf95662e251093a1b31e0be32f9429fd3605.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/79ba368c189e3202a7c502e066f68371a4913651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/7a89afce4253b9789f5db1ed98706675ba9628eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/7b63f130b95a2b27114361e429242e94cd385253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/7bc01d3d513deb53b734ad56c1e0e56dce8ca614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/864f15a98e1261f12394fa05fbbea706472800d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/885a645c3bdbb22d3982ba1796bc78b18c62139e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/8c5d0458dd86314190f06d1de242380570b9abaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/8d98a0d16c79b546ca9a4dc70d2c10845bc71306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/96b58066e0d96b28774a9a95e0b33659325e4fe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/984018e4e54564e552d74122e133a55c2986eec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9a56946efec67208a44e504cb8a3f48c446e6cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9bd08551ba54c18ed4ccfe59091f3696676b9e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9c8d312d2f3cbe4cfe897f6e0556d7aa842d6ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9e6c696c635b3eeba4fa3b86f969f7d1580ac88f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9ed3f7f391fa9fa4505d9152677a34b9b909acac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9f20fce630c753a5da46c5f1296079f6a9e990fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9f5dd7fcd86ef95ee6ef2c23934fda14692fc36a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/9fa9b0698371c03301b4743bd8cad99362c337cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/a1857afcbd13f39ddca729e6c3bb08dd03613604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/a1d303bcb225f5d68dbf96c0f54b994a97452455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/a405fb311f33373b48a60a159bfd4529b18d0afe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/a60eb1d04202c64b2c7546add3c1ae3e318259db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/a8ed0bf9ba0abd6b818346ce431df92ecf56ea44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/aba48f03e24bc22eee00d7e34a7f83f10ec27255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/ad33fa9412541ea52d76041bce30d60fe6261472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/aff4f83485e8ae5da853708965248f34ba2acb06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/b1fecddbb34a41c35af2f970ec02af5377a66626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/b8ac1c12805441cdd5d8157e17e998e003819e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/bba27e4ccdfa89c8e9fc0d8d40849ee3a23e556f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/bfc8dc885759d6a1b7228288d3f384f768b4a64f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/c197337a8b0a3db5494690f000574dc91e1a1781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/c636d395e4db12dd4b11e27cbc0ead48f98314a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/c78eac998b259ac6c649a99743b231ccdbdf6cca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/cb6dd116245b01fcf01e9a90628037a254bf2eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/d824a1d4b71d1e4e2310ebef782107b81a93ce7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/d8c6f1191b8d0a1db425c92706f01d4018ca2b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/dbb392da339f1fdfadcd8b233c3ca84be7b79b09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/dda8b36417a98326c81285a534acdc66526ce1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/dea441ca36fca85f9a5950fe5f78ca0648c3755a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/dfed0840a332b268da02f10633c877207d66d65d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/e21b64493b4a47f2d99f29e2ecb9fa702e7a1fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/e75592773d95ce93d4fc95ad8f7bebeb2de2cca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/e92d620825b4607dd33ab9f3cdfdbe9b104f0fe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/e9411906553dc093813ffe06f5059683c23676b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/eb54b266e894e7f4c2141bea0efd84b72a6181a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/eba5ec531844827bc94b3b4443f96ce20e4cd3b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/ec40b945765804b828f8003145cceb5a97275346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/ee5a647dd9c881a30bc4eb5f39274fd585b0ec3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/ee9cd411306de13dabd7573293b2142d330303ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/eea4e378f7424811272d3b49a10e3212eddc0b00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/ef9aa8c95a6dba390b41abe80349ccaf5d4cd988.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/f0ae83c35d8abeab7c0737683c37e9acacd75cda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/f321f18fa2b917084c7e5f322944cca6ec699427.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/fa37d42cc016d4600b2639ebb15cb35e1fefca67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/fff44dfb9d9410553c836d5f313c2f50ea4cc203.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-13 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-13/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-13/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2017-17.yaml b/configs/cccc/mixer/CC-MAIN-2017-17.yaml new file mode 100644 index 00000000..d9ecf6dc --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2017-17.yaml @@ -0,0 +1,756 @@ + +streams: + - name: cccc-CC-MAIN-2017-17 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/02c7c648317611d4f5597bed741fd9cff8ef6d7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/05bc1e23cba40fca7f32abbc7d828fd64597c21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/064775f91f18dfecd8352ce6692d0ee7f6479855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/09dcc4f1e5ebacbdabd7029f90a67be38fc21ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/0c2912aadab4791453ead24e59eef0734151bf21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/0fe5b90428527d1b19b7a417efe2a927b5f88984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/10a6941eb9f5316ff91065e3a7c0efa6261036c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/12fe88cbc804015864dd2b5f41b03bfba4fd5a2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/132d7ac30acdde3b6f8e94e8085d59c602950cb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/1e30974cc270bcb4b92bb2aa8e3a0c02f3c7d687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/780fad3650fd65380d737b52816814f30102f33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/936f7e395ec243f41653a60439d42305df91d2b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/9af60dc5b75f4d9b321e2bb9d016e02df5853788.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/a16c105d0f89155dafde95ca50aa630d74af79d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118310.2/warc/e2839c4f744bdd4b8f383937350310935f9acbbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118477.15/warc/3f3cbd455ef4a78ea2977dc4e781f3d3c567ec52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118477.15/warc/862dfbdeffde55a1df8d919f1deadd14ef74dbd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118477.15/warc/a5daeb1d9b35dd41aab6e799850e6bc47e5e7e1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118477.15/warc/d4659791e909b47d7af03b22538306959c054445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118477.15/warc/e8f1e098891ad6f7a895d18746a8b94812c299d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/6191ff483b6988bf209dc02b465e3cf6b102041f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/8f1d5743d79df8d2ffe481393a7bf4e427f8a4a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/a27f38db07268d4a53b3cc81be1521d72825b06e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/ae2fcc89bdb987cb8197869deba48e0066687c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/b80b6e3ed7d8fd7507ea95895411774806aa8feb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118519.29/warc/d64d7c333b7e72bedaa8a5491f604104c2fa1617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118552.28/warc/1b4dfa3a88301a0545a431c878d8da7991be85d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118552.28/warc/1b71a02b1adbb8562f3fa3f3286de9bc61620bfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118552.28/warc/5282932409368126f40a5ad32a816a0dbc8a3dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118552.28/warc/8ee4e71735002947cdb61a4411074bb19d250d9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118552.28/warc/bd8d5c8515aa589485e513511d854031a0faab9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/3a159571ba6f233662f91ae60d6b1a7866078c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/400a6da15d86df0f7a23fcfe37af758eaff8b020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/b0a3bfc1b6d883ca84e87e3e743909e7564424f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/c22a328d0ac06c7b1fbb9f000f40a7f180093bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/e9a82827480fe6555f38c50641993cee47276f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118707.23/warc/f5b2c6a488971b282c0e488deee87f12e7d9b134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118713.1/warc/21eda8b41051b1d256312f3506a685e6c25a69ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118713.1/warc/367b5196622416ac28b8c0d5735cca3118893831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118713.1/warc/6afc618dc3dcf0ec4ffca64aaa0dcc99b405e7cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118713.1/warc/855421ba62eb0409e12eecc92d3e1827b86e6385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118713.1/warc/ff9a688b52052070661804d496c35926d349cdc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/31b80d0b0df54a2a2c0c5f3886cf47163a0c54b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/44d679ee8f295ea173c6754862ff072a0bb11c3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/9553988ba93d2b8d7e8fd8d22b7985eb26d0387b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/965e12d8af593f189d8248d8fd105bb093407fb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/adb953c3bfa37df18aa12d295840410246c96d75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118740.31/warc/f94fe8c8966da838d4cc8755b94b8e6f2f764895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118743.41/warc/139b42c2d154a3da8b3a540dace7fd22570a62bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118743.41/warc/9f4af231595cdee54561e72cbbae05bfec759eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118743.41/warc/b96a82792c0f2049f49055938a35ef22574ec6a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118743.41/warc/d6d5085f82792746f459eff6d0f8fd940b516390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118743.41/warc/e3585f202a7e82187a257b44b34fe303865e083f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/3f43d0a0fc67099023cc5d263fbfd16d09c01913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/a9abd0bb82df53660b2cbcae23a807e8cba4dbca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/b416ed64f5fdbe21c165ddfa22b0a834bde35deb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/d8003e06632714b70257f12a701b51831fa61d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/dd43297426b681d349de989e30c0584b85ae9b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118831.16/warc/fccfaa3e588f216f694d23914cb3657575cf52b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118851.8/warc/298935d030feabc95af8e52518c9a1d5d5609d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118851.8/warc/b2b20a25c249f6e470131d31b82be11368f40056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118851.8/warc/dd0dab4c6481e3bdf44fc6cccd15d3c668099a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118851.8/warc/de16ea3e15eca3c015690c5831cbe326b5ad26f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118851.8/warc/e605ffecfe9a425464d126920721d03d9aae94d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/45c0069477fab0c71c1a7c2d7fc198398bc92c3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/644eca93e3f886b3f8e93062e4ecdf9cda8038d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/73c4036065c8e4f3057df7aba86f5be2835a2f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/bb79966fb34db4e6addc556ae475b979099a4bd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/ce2665d462abb1d9881860a1926323e8c1fba078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118950.30/warc/f6f453304753a9bca74b0dd1711e6649962fb8e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118963.4/warc/1c2f5536a2e0834ea72cd6f1ee1e8e4d79622bdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118963.4/warc/4a65af0bbb5544c7b3af86cdbc262c7516a41590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118963.4/warc/80bdbb3bbd982f83383f96033175d3e874f3d6b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118963.4/warc/b95b02880365971b8e63aeb0fd94eab9d27e0000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917118963.4/warc/ee4147ef2b6cb06cbdcc9c3f763c6c11c90b21da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/26a118ed3cced4341d27a6752fc764f20f976f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/4392943714bf3dd08bca721314134b2b7876c339.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/709062d0126eff80c4395b78be125eb59abfd146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/a7c3324067de7b2b19fc23b0a63831a3c5b8ce19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/bbfe5a92ff9a6f5203094cd7e107916979e7ee40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119080.24/warc/c99d8d5b9558c23a83a155158f37e822bf2ea8fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119120.22/warc/2d265fbb49e50948ce3dffa7a942abd9c8b90b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119120.22/warc/3675eb9ac6cd5f79adf5648de394191ba60d6186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119120.22/warc/5c2104f4b29852664b4851be77430d7eceacf8a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119120.22/warc/8ef6cadfa5c9f6698fd13d9d041bdc5c6378b64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119120.22/warc/ab185550b97db9f288c2bd3c823e71faa86f2125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/4120293f50e1eadadff5cbcf4f447fb1b403a182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/69423a56df200bf2679e897ab55e2e0d42918880.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/7c65fa0dbd71235b318708a964ff5132a2dd8d65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/b27b46d2a50e3b8478fff16e6545d47b74f6c605.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/cca2370b48d9c862a4e2374823a11e0b999f5993.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119225.38/warc/f0b9f68b024a121702f09b3301024b11ecfe0660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119356.19/warc/53394fa6228f4b619d73010cf8042b25a68a1af0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119356.19/warc/58b22adcb8194de4f2684b8f5e1d90f6767f2dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119356.19/warc/9242d6378e00cb07cfdf8ebd29b62e555f8d9364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119356.19/warc/a72cdc863184eb7939712e81816102f379a0e703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119356.19/warc/e9d9a94858b9cf8b15a8a97388f201ffbe18fbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119361.6/warc/30a693e997b03ce75436f4fd198d0f92fd669769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119361.6/warc/8f5e55eda6361f6a524da728c1ba853b1358705a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119361.6/warc/8f9cf0a14d4a66647563c89c71b118cd75eb9899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119361.6/warc/a0a4dc9c4441ead6c7251c19db58cc29ee7b4e91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119361.6/warc/fa9355a942350960c23add384c852c6df9a04c82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/212d3378a4bf0975ffee343361233b9f84d87dfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/409d5de7b2530af726b4428e951059fdbe079d66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/66a49652e51d667f0f7889db2a30b41c3c11d57a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/c7c67d1e0c2e0d49ad5f43d2fde4e42ad578d574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/e0e461c7a3bc2a28b906ab3e65623d86ab987e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119637.34/warc/fbe7ec8219aefc23f6d272fedc58c287088cc345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119642.3/warc/1a3236a41901c80ccdba590d735ba84546073610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119642.3/warc/334017ced465299ea2560516c272cd54cdd30105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119642.3/warc/44680bc56971f862f21cbe246995f32811c3cbd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119642.3/warc/4d182c1907bd155c68fe7884a62b04d741a75e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119642.3/warc/a3d0a89816ecd9667f2eeeae2d6b45bf87567e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/0e3d869fe1a7e06486857f1a33613852329d5791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/299aca65ab386e68c721045926b7dd00cedb72cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/52465d3b41bc1f836bb30ea3edcc7eb8ddfe9146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/909ef8c9e3313a1820dbfd2f8da809747e0dec79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/9369c0db3761a275901103bb7b76481f734c3b7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119782.43/warc/9995c06e18b225fd3d8bdba08de57dd7464f868f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119838.12/warc/1ebab772957937b97415182b47be715a14655afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119838.12/warc/2d66a43119c65dcc651260e6dda0679426d9936a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119838.12/warc/4a58cb56726f41c210d58b6c3a5a413d6bda504f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119838.12/warc/6ea0ba60b06ed3d07ba700cce1d523b4fafb96a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119838.12/warc/85d7600f8642180240fc1c17838537f2bb0df219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/2d57009a3b66798b7a030d818309ff7f5e284e03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/3eee26784abf8fafc750024c739377e8f3afe09e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/58300d2049b9f5db0ccab41be7fbb0680eab2bb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/92bae70cbd595ec75dca41cf4e93b3254aab8fa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/cd33004f84e1d9413fe05cfdc2e6dc3e96a92fe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917119995.14/warc/fb8daf54689a8fd269abecad22587dfbee362b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120001.0/warc/0d8b20c451e9f34c0193a512a7b841a6087178da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120001.0/warc/3db3816b56f295841022f094f055c671f0c40098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120001.0/warc/5b065042ebefd54ca0920d58836214a636394d0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120001.0/warc/8b73db58ba30206ff9ecf49780ec98adaee46c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120001.0/warc/8edc0031ebea444242ebb21c813a3d7020205eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/167dccd11f58d516970f2ab6b1305b719dbcfd73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/1bda84ac88945aa5959555a5d1c7cd6f65a795e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/22809db6cc69cb3b5d9e075527430134ec9a5d23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/cac34f7a818c13e05aa9b7a649531cda8b38ed05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/cad3a39c4841d252df5ff494cee1a00684799b84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120092.26/warc/ecec6b8b5aa0a3629ba68400c073d95d3e863926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120101.11/warc/1c33418910e9688a0818ed103691205056ed18e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120101.11/warc/383d0b53fcbbaeed12f4ccc9c620e7c302d11818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120101.11/warc/967a214c1efaf5b6ceb125853c5b8941bd09616d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120101.11/warc/aee5bbebb8284a7a78b6993e01b3c4fa12cd2c1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120101.11/warc/fd4653cfd784048dab7fda98ec3324c8cf38af22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/11ec66b01a926697c63d1fd06f5adc072a4df000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/2ec29d2a2395c87dbc32f8d51c28cf62663bcc51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/ab2e03dc363a8ad6ab7a092ddf8c4ffc24450262.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/acdd13b896e7011644f7f9956b25461b384cef7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/d462b4e05e19b031c7cb46ca7c54f28dbed6e153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120187.95/warc/f81c5879aa03e3587b225b04a93d3f945e1e9375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120206.98/warc/4c51a7aaeccef55eccb8b2c955c08a34e733f1a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120206.98/warc/5a4e25864b37f1044a490621cca4e9d6ed8d9f11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120206.98/warc/5b43363cd2d1e7f710aa262acd935903a6d2fa0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120206.98/warc/6f96e195d707df2e760faf0ce5a8dd8ce798faf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120206.98/warc/e7d6ae6966db54fd8a462439f70b3aee26088e46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/08e429c689f10a3d2bbe03c04a74fd222a496633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/131af115a039877ded95a4d6584f2022ab5b377e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/2bb10280290e5ec58f10c3e0467082e217c20156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/2c23cd1613d197a489fa6026eaf519671a8cc737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/51a164d4c2ad9bf0cdf3203b8e60000ea72ded0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120338.97/warc/b014a56b82b78d667987989bd939339d4a750356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120349.46/warc/3d202f85e895bd8d2f78b272c5932ec9d5a5ca76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120349.46/warc/8599eb2256931850b68d53c5a3ec84100ddc6c67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120349.46/warc/a4aad9ff131e2c189b48555757156cccfad326dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120349.46/warc/b118c4500e6cab054be4657e81e3a1de29948edf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120349.46/warc/f63b304c0eea2e826eddd1ac97193230691319ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/1e3a5bc6188164e86f35b296efade2b0a420a792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/520fc6398d387363f658a7acdde5311e64a17a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/5c60adfe066be25c99c3b30ae53160dc33fe9db5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/6a6efd2dad043d146636bd1bd1b069314fd7ac10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/917d1d1373c3e8c56528032efbdfa1b56d9a510d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120461.7/warc/daf5fd5e1f3e32236d309f538ec761b9298b4714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120694.49/warc/7bed3e5b45ff02535ba2bb4875e78faed34042ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120694.49/warc/90724e441d52db789bc7c92c6e6bff4bc2b30b69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120694.49/warc/95d9b708e45c53961edf880af9d0d4995612d821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120694.49/warc/b2c8377fd3ae68b5bddcad23562d9213ea3f839d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120694.49/warc/bd0b192045d71b8066b55edc8997bb05ea303836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/19c5c15cb40dd09db020686ad7a7e8da7bdb525d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/9931fb9d9c64bd14141e15ab21295c4130b6ea44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/a9fe6e447ba8b7462d7fc867124b5fa4e4d9ac9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/ba7bf4a107a8e59bb920b28cc5b5f53bf5160939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/df78de382363aa3c5629b8952e667ac1bd825703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120844.10/warc/fdfe0303ed0ddbd9481f37884df5319afc5ef286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120878.96/warc/15b2a9cee377d1645fe4a600a09664eb434a71aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120878.96/warc/328a46f6478c2a9345f8a74c6106821ca7022273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120878.96/warc/8fc77b64f10502457c543e7e4b8d43430fab12e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120878.96/warc/a84e388ff4f5984fc3b554c27b11caf867c67729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120878.96/warc/adb10073fb338693b26843e8e2e54e72e4ff3ce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120881.99/warc/275d646868426a74bb02f2abce3972165083e1a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120881.99/warc/33362a8d145ac88a322c5c48d9d39bb4051b9fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120881.99/warc/636715f02c738a652d9e0eb74c051313a52a7710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120881.99/warc/7ef50f1273355ecdfa387da302698c2046e59d59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917120881.99/warc/b0dfed67dc116b10003d92071891640f410df815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/00b67c10bac743c2baf163242c8de6761081ab53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/0b38d4a5c727e65b4644c436783c3fc49b145257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/8a18fabc5042b3695cd0d893f8f47c4dd054984a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/ae988f2679b668c0bf1370ae63e5e8b2f42659cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/b959db1515ff77f0cb98bb04e2d22e92fe9c25f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121000.17/warc/ef04fbd091e295ffd69e051da711fac91685deb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121121.5/warc/78a8417a1a190792bc709d13cface226d497fbfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121121.5/warc/798295374a8502dc4b5150cfd691195f30353ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121121.5/warc/88b668f5d6442fb3bd8e98008e1d2bdc02039068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121121.5/warc/bd131035c70543cbe684982f5744b720d9fc06e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121121.5/warc/cc0b5fb8f092b60f49dd0c6372f346ef67c2e038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/09d762c75454235f61bc9fa467d32ed7e3ef059b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/5583718619bc0b04316100cf4d41ced1c48ea36a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/b14f55f5f4010d77f463279a1e6da0a2ba1a4f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/c0790b695a3b9603e2ed2345582fb024dd7ca881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/ebd68d957b1231ac60fb6c39225d8aeaf484d094.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121153.91/warc/f0a07381d1bc54324c112ec4ca477a802d621302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121165.73/warc/440eedc06c00867424feee6a53938e71db191fa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121165.73/warc/9db274a3d42bce6af92f7b3a9ef544555c135ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121165.73/warc/bad2f4558fbd28c195cec921b95c4cd42b2f90e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121165.73/warc/cbf994c659850ca2a8c4381b893e374150ac5ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121165.73/warc/e966133aeb0b62acff1ca9219f71445978627e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/2173e5241aa7479665414d826a4c702c8dce0d89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/2a7cfec269568975d6725031034b511074b54e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/6137348cd18158f1de3fb426e3f179d90a5204ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/70aafb780bd5674f27538ccedeb9671d770c0405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/712d7ec26d1258cbf1b5a972f31cd2cba6e863e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121216.64/warc/c20018fd52090356347eff5ddaef0f819892ba4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121267.21/warc/3b871e3c1d69e74254fb0106302761e1372f18c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121267.21/warc/80c3059368f2874bed7c066d9aae8d3f811b5156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121267.21/warc/8fb152b36999084b2a9500677dafa037413a00a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121267.21/warc/9d3063eba7517a24990d29cc1504ca8c06f7be22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121267.21/warc/b5779c8da0ff28f4a0240a94f778c4b683261aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/9dda1646fd87cccfdd4cce5f7d9bb289519b71e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/a7929e54844f9ee7a4e7aac2a66353b469c54941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/bb6793a7fbcab2673cc8b0698136cd41741f2e8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/bcf7ac75f999b92bc38988c3da74e92da087e2bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/bd2850bd6e709998116a6724f4952f52eaa19989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121305.61/warc/d477b01c0f1bb96c0a13fa43a20645e0add80513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121355.9/warc/0b66072849c82ed30e5814f2d24547d379d47730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121355.9/warc/2d6c7284bda557461f61b236c0d0c4262d978b10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121355.9/warc/7e61e5e790dbe18551db610dff2f438d6feda5c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121355.9/warc/ccad15c550985825c63d96519e2351634e4703b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121355.9/warc/f6251e0119a4a431ea6f55a455ce36a54287d21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/00ff170809393ba11b23cb86a9c34b3b45fc0c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/0648ced50dbaab4f79bc2b56e93805ee9677c1c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/0e7995be3141be5a6755c8a49e5d83286b1dce17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/5d0560885f4ccf63ac7f518cbbf8766964e6a7ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/676d93f2c6e3646135c0f68db18eb269b1a15670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121453.27/warc/8ac2d84743d734236a7dabafbe60b30057e84ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121528.59/warc/4223ab9371b514533380f2a256c731ed6d309fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121528.59/warc/5360d31accd7505d44103bde9f1c664e38e949d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121528.59/warc/b32b6fefc9feb5e02983a09a874c1e3c8d09750f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121528.59/warc/ee3c059ba5071570e6a3b2aeab3f51c8a0c12c00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121528.59/warc/fcdea448d7db217b22a0016ed6c1c5f45ab6c45d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/02ea7c86a2c8c149aa1503fa0215490fa8ab11df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/571c9e0f209d53b0eb7b80896459e26c6e0e4939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/805f5bfd8e0c78d31b9f98884b901477042986d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/bfe31158c36d3777c1cec56d16b4ef223867f7b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/c2a2f388ea536630e434e43a68bc247ea6c00137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121644.94/warc/de946c624d5b20959365a24c82c3ced6cf0e2d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121665.69/warc/26725008b7f5188dd44163a2a2e9e3018e792f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121665.69/warc/3ce94fad019e7d79f2a56cfc3429dec605c94623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121665.69/warc/4cff4c80c1d355f2f0b091638b7980760d1fd9ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121665.69/warc/5f1ee8ae67240147216f0737cafea0d5446294af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121665.69/warc/7381d46543ce0230b3ba37a391a5e58a0246d431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/7db8e3d496b0c3dae6a435f56889e835c66578c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/99412e4ac9e7d46ddd6c531f783c1f58f03a1aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/ad458361aea5aa25f5786af81de042a30282c720.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/b80067f8c9c07eb3f8a754b2527afeb5ae4d58fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/d5b0a7ba3152d7b6754324f15f2a0cb5d474f357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121752.57/warc/ef121cda35c3da507da360a72d0a9eaa01f09669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121778.66/warc/58e4b55449797ec20d7c49050ad3ae3198f88d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121778.66/warc/63258926c9fefbee6a6c446a0609569b0c86b40d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121778.66/warc/8d96b2ac401d9ce03788e9fb084ea0a41baea7e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121778.66/warc/ad0d9f8ec0874f5d8297003fb27a09d039ae534b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121778.66/warc/f9c3d1fcfa2fd1aac08b12f834119b509bcfb026.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/6079372f83f0a29d564d29297307174160d82995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/6961d496df861562b2c410c6fd6497e3fea3954b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/860eb4538c75be0774269ecc2d617b8fb44cd63b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/af3ab9539f9adfd424d928f89b65f7df3333001f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/cf6e36ba14f57ee856e067b9881042891b28a5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121865.67/warc/fa1488f9bbbc4bc6dd119bf2bcf090a955967aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121869.65/warc/31843d40239abaca0c2e1f932b8d03b5011e275e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121869.65/warc/743cfa1b34844f88fab9a687c09c37b94ceb2fd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121869.65/warc/9028e540086ce8277f1c12014d5f0b2c7624e9ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121869.65/warc/b2137237582ccee1e6adbb570e8238c2c1947826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121869.65/warc/c6d82d0c26cddbb357e647d0b9886a8f2af4a981.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121893.62/warc/4abe886e1ae71f2134690eeedd356f5790022157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121893.62/warc/6f6465337450a84d4f995cd1b85e3270171f663e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121893.62/warc/7c250b69acc0ba9fe4fd87fb51d8dac0aa32de6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121893.62/warc/b4dfe3faf4163a3323b016d778bbe6e2e81a935f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917121893.62/warc/fd91a6adf314a97572f89f00b34ac90f5bb09fc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/165c1ccf9b40ba8520f0220c847866c32e1f36b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/51552cbced513179d68ed2f724ca5ae4356faebd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/556d0f6d372e33743365d81be344140715709786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/56c4a91a34b3bbaf471f740843956cea8e73e757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/81f0bc2ae2c88c05fe42ad6b473f09636ea7c64c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122041.70/warc/9289bae4f4b15b17a40e16633a3f51a03d4f1b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122159.33/warc/37dc26389414e39ec04dbcda9c378487075c874d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122159.33/warc/8980181e14f3879feb23e6e18f0fd2ae0c013abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122159.33/warc/bd16310e7ea7c34fd5eeb1981870d4651ce7da74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122159.33/warc/c221c732dde859fa19f75bd07e0ea44664d744bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122159.33/warc/c66489d570c8095b2e2a2de728af7ee809b6308b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/3239dace818c04952119ac697d0658a7a18b0226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/5556e0e6c7445dc411c4133eadedebcfd4dabe8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/8d6526b74eda858760f0952c8e12e607a2dcd011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/90c14b9249f30538aa8f652e00992d23ca8d77df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/e7a92249d1f6123d47ebf8a3636d097460a5edb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122167.63/warc/f3f802d11ffaf22109ed10ce2c1318b18074bdbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122174.32/warc/579ee78bcab1ca0b7970ca3e486c1593c14757ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122174.32/warc/85d354b86fda27c34d6a86d46627684f7eaf9f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122174.32/warc/b151088710b74953ec182e8b05abba3a37a3bcc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122174.32/warc/c5032d7ba02f0422624e36bd38bb775a5b8aef89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122174.32/warc/d30e060317322e5fea297eaa560b4cda70c5cbcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/0eccab924b40173dca53aeb90baed70d0ab1cf0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/18258eb3c87e2db6b84329ea9f99150f33e2d783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/2c0a6c9d7ee0fbe100821919b6e60cc337f75c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/bf47764fc0d8e28b1d6e2a63d1909a00d616249c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/dbdd8b9ca5939064960282d6f89b56e9daba4505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.60/warc/f7f64ce3c4d4236829e25df86fbaf6d91a745e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.71/warc/56f6be9750159b638f623b890f0a9cd9a2934623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.71/warc/9ce6110e06848a0b099eae818dc202b30b441eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.71/warc/b0ab40cc812e11362e93cdac5efca2be36412b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.71/warc/d1b46ae775ec0f7dc1ac102e36fa44f29cc69bda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122619.71/warc/d5e49a123f72e4d87cb6646572d686619a501b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/32484fb07c6a06d360ad95b85d4fb737bb2425de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/5e43698d2f3d9d86a4167cc2bebeed235541ffb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/7c8a6a3bc48b78e574d80d1f3f16be1bed191196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/aea04ac30989216280f6bbdce8a415ef31d75658.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/f46e209a559f4a000e01fb17114658aa978a01c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122621.35/warc/fd562fc6354783116bf2f0d71e5de2ed69845454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122629.72/warc/0feb5fdfbe2eea150a1c072179d9af760cc012f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122629.72/warc/1950c6566239e31b83f64c05ef137701cd9253be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122629.72/warc/23e0d1ac9c776bb022f151e522e830b11d111d89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122629.72/warc/2a50a960055a2f3d5956e75493780a6600c1cc12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122629.72/warc/5126362fc34b6287930682fa1be04b4f3087cef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/1246650b7a88a94c7e16951a2f7a2aaca62d37b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/2d39b842b1b6cbe3880a66eca257eedd0f578b6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/364554d1a5c4f5955f6f607180889fc2c4e6a122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/4454da0ac115c7d4a719dae974b02fd251f4e688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/e8d14b63fa568585bbd56a2a9c204da4aa4979ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122720.81/warc/eb6281dc1b522601d9e958a77c85442792b45978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122726.55/warc/822d4ccf3923067f248e951646870ea2b4770512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122726.55/warc/86b0233c095f5bcafbc3e2a04d2e9d75ec3fcfe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122726.55/warc/c25956391168952c94da5e6bb3fbc75002e92545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122726.55/warc/d038f1c902ec31921f81e5700a96ce37fe088ae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122726.55/warc/d8b223f3d8b4a42d88d61dd1400b08e4ff0ec384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/0121dbb0e6f1e3cae369fd46a18ae1756b3cb49c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/115737c79fc38dba784241ba8be5de3e1eec7410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/2c33a30582768cd467b5fccc791569e00c009411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/501186fa10c226ee5cf6a0f540e39a924ca2fe9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/5724211c5fb73e785d424a0b3b875bde2f141525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122739.53/warc/982de690905c9e70b4a881e0e1f15a8cba09bbdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122865.36/warc/3ce1c148bbcf7a2e41ec7abf9fd49c53527c8658.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122865.36/warc/8b5e8a0302e132de37419600c46f9731dab20063.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122865.36/warc/abe76cbfc1b54dadc15d92a54bdf36a650e2a118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122865.36/warc/e69de5eb9de7f9b765d09e7f62ed9a205b16d63d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122865.36/warc/f769bd89169427713185e48b4546b1fac863023b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/23c38e97e26bb57da3ac5148fc36f0617ad74d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/462998f64f381b94194b54633511a06ba18e084b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/731b7f819b7256cad2af2cc58bc87499e70d775e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/833d7b7a6e94cbef0bf897eaea427cf4c98e13cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/badb3550578a594704ecbb592a4cf8bc65ca8ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122886.86/warc/cc25d883a8693b83c371176d7cc6ab78fee51c90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122933.39/warc/37173d884a46e03ef5420c95988aa2ccc56f22e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122933.39/warc/43c03715b14b7419bac164ebc6f64e013820e7d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122933.39/warc/4d9bcb953d90a6ffc81e507e95c54c7fbbe97a9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122933.39/warc/793934979c937b56e9903d1add265bf748f9b86c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122933.39/warc/d2ec7db0c1b5db7947998e39f5b31d38fd630ce1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/428aa845bc951af0699b710152c3d227e95f1172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/57c4dc8faea348d9c4f53bdff3df3473d437461b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/90fa9cb3aa208ec0236ce0dabed6f89dd1bf6cd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/9be0202b241ccc04c16433cd07d9eda33eeac24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/a4f5b850999fd2b59e8e36b2b14c2c91a3dc325a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122955.76/warc/dad588b552796a91d1835b615578f24cfb5f08c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122992.88/warc/a9680854f4f895cf2c27ad3e117fc0fb51217ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122992.88/warc/aee63e81030ef7b169c6ecbbaa1dd046e9259db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122992.88/warc/b70f80a827700dbf8735f69d4b9b800db25ce803.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122992.88/warc/d7cf5063147793be228bb87d3c6dbd34096c30f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122992.88/warc/df37e1905849148c7b362dd2e2dc65b07e9e4bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122996.52/warc/17a6a1460d82ce155a919f145486b45b368c5bc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122996.52/warc/4685a65ecc65ca64d3d79bfda0d13d29d86feff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122996.52/warc/795a2dc839836294310afae50e64684dbfa730ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122996.52/warc/8c56ffdce733ff3c3f1e8846757fb3f6979cf1ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917122996.52/warc/a8a40b4c9c0a8388ed019be427e38a5340e11c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/219f9b231a6a3082bbffd88dc8452f3f382d9335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/2fe1ad1af2f6f49ed58ce317765668442876c83c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/57cab6ec81d12ddd0ee52b9171c0f4007c622cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/923700a4a919b094b09cb06a1dca9422d9a40890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/a2cf16c494e22f5e2e532b38d6dbcaac00d44c21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123046.75/warc/a4dad3474c5b46bf9340afa29f5292b43cf82726.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123048.37/warc/215ec8e4fc358744eab735c418083829f7a25435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123048.37/warc/2f0f9c124b54583e12271dcdf74658feed6cbd91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123048.37/warc/5941334544478e5745d263112dca6e57b9b44b52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123048.37/warc/ba7970d0071504d0d5a901473e2e8a1a5aeb7d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123048.37/warc/c002114c388c68ec92111097b4ebbe7c4ea3bf7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/2921e8e51a47343b26c6d56dc7efd253c48099b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/608ba758bd1f35b74f8db5de617a1c749d67d8fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/6674644da93338e8d7e85cab93429093317d0e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/a5eb11975c56e34cb0a4df482bf1529aa22af0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/c2be0c8b823a564a9b78132dc16e428082272b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123097.48/warc/f25a0c463192d8ac11f6ca2877a1841d179a36d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123102.83/warc/4f20b7a2bf171c4350c66ee03820581eeaad502f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123102.83/warc/53bfafe55f64bfa12213c9d7cd6f429c74bbbf4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123102.83/warc/5a8ccaf3768d50562e22edeefa892d8bf25d306b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123102.83/warc/68ec74c5e9e21a85c831682a294ace2580cdf566.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123102.83/warc/8b6245898d1e398e8225bfec52f7845d590498b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/45fe0dfedad9c1b5ea474bdd81e5acdd89bff19c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/4fceacbcf89ff6c34ffc0e1a4231d63d2a06ee70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/7a500435a7b1fb2c6eb1953fc5ad0121efc0c229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/aa5079cf1f8477f08ca6933a3804a4f0e30d152e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/e567369ce85870527c60d4ed7d0e973d5abc6158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123172.42/warc/fcb95a72db8c4dd3830a282d074d12a04e57da4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123270.78/warc/16df943dbc9a61917c70f39b9ce0e88544582612.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123270.78/warc/88ea5db35a73f20531bf512bc60735c75c18acbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123270.78/warc/96d2ee1fa14a7415d19499bea19f813ba3e528d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123270.78/warc/b293d264c172984b1ec3efdd24bd4d22a3ed0efc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123270.78/warc/cc61ac2a4ed19600fcde447ee5391b61a7546e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/7da6266ff6e0ced802bd4063d5785aab850ced72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/844a6150025bb37e3f3db3f67f0140eb9d5cc0a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/cd2db068da3ef1788a70c40bc91771bc28b126a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/d11a99c4ed38b4e910a440164a9cf001118e1fff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/d5c99f13725872f39241966fec934f04a0136bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123276.44/warc/f07757df11c0029642d85b6d2e945af25cb390ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123318.85/warc/1a1297065d7e632c923fc3d42d6b269744fb917e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123318.85/warc/2f7621bad8c46a4c6f50a8c9cc7a0d19c2e1e551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123318.85/warc/af35d0f57ac290bcbadbefb7f2a306ff860729cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123318.85/warc/b4bba600670832d61bea8969654da30e0a980cad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123318.85/warc/ba1df2beb37792d1958439854376348e05b34e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/2a655b7f90f8ec6f82133bb9eb23b90664382785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/59ce97aedd65fcc7c287cb88859c366b50ae6d2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/9c22cad2000ae9d61ff0330d67bb020b86e6ecc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/9ef7af2fff00356a648b989cd55900d462006b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/ca9d1f38b086f292de592b4cb73ab03d57cfbe75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123484.45/warc/e0d1cda07fa31f906112306fce0dddb1d0b128cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.68/warc/13bcf53f8c0ae0a2b712fd5dbb20eb613ee6baaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.68/warc/21755a1c2d68a9d739fa89f8033e479d9862add8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.68/warc/5f8d771acb04987bf6877a89588c145580529c36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.68/warc/a210b47aa1dd490e4df5f169b6812e8a4734b115.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.68/warc/f789f4a463e13686e60b7220e9c4c4dbc6dbf7e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/123ec4a78b169adc476f3f0e6d46a1dad6f1c473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/25f4d6f733afb4580122df0827f8038e8b487f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/61560fe07d5ce786f41aa3949f1ac30e3510d897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/845a15c77c4392663ccb0b856c19cad84cbbad6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/c53d9420c8a6d6bbd4705981013f5ba8da90d5ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123491.79/warc/d2156534cffd3e16ca8447ef7be8e7893dd858d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123530.18/warc/6fe3b0d8dea5e7b64386bac3626bca662d4cf56f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123530.18/warc/91870a76abe2517a9bdf5d168fa7e162fb8b82f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123530.18/warc/97ac5c7735718b229d0b8286082e43489f32d856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123530.18/warc/d6583c8ab6f2398778b4f0acf5b16e09df116b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123530.18/warc/eca92a5de40f9ff2379139018538bacbdf4c5985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/07225c86849f61cbdbc09c4562304245568fa56b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/0f63f60885f03577eb96a3af2cd88fc4d378253a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/4d09e89d17a07bd4c61b8a26b12e0b84b9532def.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/a5ddbab72663df9b244aca1080f5dc2a261f8ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/a7e109534b0b832eada0c478cb6aaa03a7860ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123549.87/warc/b7405a5b5dac3d850130d4f8e519a6804828ef46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123560.51/warc/0382a1bebc387e4c09d57f98ba2622ce76cd1d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123560.51/warc/191585fabdaa1674a1f38f2dba3d22fb848d0911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123560.51/warc/314405a07ef917545975fbc286948ff7af6b45ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123560.51/warc/a61b16225757c721c7224f495d1a12c9fb3ec704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123560.51/warc/bab0a479d9f8b7c49a3ae5d838b21541f06dd777.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/1714ac9943edfc43156308d14a017f966d95d18d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/9f42c72ccec9ca8411cb80c4269b17769c3b5e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/a5836083e6d0eeb0583a7f7b4605a30cbe4d8c68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/b3e67fca0cb3c83862a14508d46f9418da4345a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/d5f77eeee345865000bc4f0318bc6028efdd9558.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123590.89/warc/e9fbf8e4d26999e0418d966a0c129afad72363b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123632.58/warc/117dc3d610c0c6d88cbd13ad562012d5a2c5f4ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123632.58/warc/1199055806469d578b3e7013e10e5a465d286eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123632.58/warc/3f00607595e06b9c89bc6e78de7ba7423c8a5bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123632.58/warc/4d48a5c45208086bc761714e7a4e20440f54d509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123632.58/warc/b0ee81d692e7a9873442f4a545b64895eac191c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123635.74/warc/405b86d3fd79ea211e9559b1fd53bce824314722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123635.74/warc/73ac785786f16ca0d482564cd815d2b0db264fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123635.74/warc/86519d6fad958871816ec7cb7b135136463ca252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123635.74/warc/ba840906280d822c3538ec2e364474a34d9504aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917123635.74/warc/ef049ff4d48a5f4c30341e91012d65d4bc16a64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/1413fe05600eaac528495c2d0046294447fd726a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/14255594053039dea2e185b6609d89a0c935944f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/2b899c6b89ad8298a378ac10d2fba136c53eb66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/4d4f8749ebc72694d2a685b7d8aecefaae3baeb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/74c0588f5769c281f8bf7ba07ef17b317cc2df17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124297.82/warc/ac921dbe64deec40f67320e342bb7a75f4b7e916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124299.47/warc/40d23872f257a720ea7d0677ff9dc8b0a5134dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124299.47/warc/49f51b8d19337653811fef45113a1a12545a3387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124299.47/warc/549fb8eb60eda9ab53fdefc0dcd2935b50887ab2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124299.47/warc/990025c57e735e8fa57f74c2eab26698632b2357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124299.47/warc/fcd4578e5c79bd9f00161a42431d9cc688c4712f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/140de82d295bb305d5c9ad56c4d0c5475128d26c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/2c013882ae9a9b388a218fa2bb76ef62ddda59dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/4580ed9424d19393f7320d0007bb92099a8c6e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/9338956ed188c61cc96f4e7098eb93d048d38d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/afc752ec2ebab1f7b3ea1436f80e56098a60033f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124371.40/warc/de4435cfcf8c9a739080aa3aa3b25dbf3625a3cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124478.77/warc/0b8e4460a6ad10398cdf783102b380f27f674aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124478.77/warc/6fc4d6fdbd66c67f7d5e5617646920ddce43cda0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124478.77/warc/777c77523ce19c7c0d121b6e9944debb7f6c92d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124478.77/warc/a05a460d8ba82d70b9b2119e796337dde13d155e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917124478.77/warc/d940d393c1b17a693b7fcf03225ec9cc9b908884.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/0cda6cbc7ac2f5a38f3000ba2094e27c96f0a0e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/4bd2969a523c187352266645f799691037cc47f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/513c8a7553406a1f82bbeee5ef26e8ee2edf023f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/b15853a61e7b56b8624c8bf8919f73f481923618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/c385c83f4be81b28d9bd465e79fd9d2eee9aba9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125074.20/warc/f455d3c42a4f7204d1c1af4c507110138050a4f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125532.90/warc/217eb9744f8e92635610eee95a0641af7c008df8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125532.90/warc/3791038fff09f881e23460616a7e6aba3d8f53c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125532.90/warc/6a0a7c8a6e455ca5f8aa917880c17fca6dfd055d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125532.90/warc/c1185607664e2cbe59ea88c8ab13fed3eb269feb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125532.90/warc/d84718ec38692e9c0c69b913824b8e541a19f8e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/0f86e87623a7fd1f1e1703de1f53d308240ccbdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/10b01004e1e8bc1523d1eff3cc653006d89dae2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/76c38b9147be727e79180a52e669e217dd257bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/7d3e7aa3b99e2d0dc7747a58fc8194e84c90f652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/ba6ea1982ae51f8abd0be46dad03e50258efdc56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125654.80/warc/c4c2f98696228ba1eec903e1db5472931998db84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125719.13/warc/204197ac8651ae88bb7e9f4730728e88dc1fbbad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125719.13/warc/8617dd8ff41bc109210bf005df1d192a00c62631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125719.13/warc/a049f9a1da93655e5d303b80534f00db5df3a156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125719.13/warc/b59f5638cf3374d8fa1527f3625beedd2489bd6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125719.13/warc/f1e319998d7d1b40ee1b0a98751d33dd683294f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/0096fd557eaaaa3db5f119cd825e718efe5dcbc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/5d7d2d9c7fd6eca1405e46e53cdc601090a2dbd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/81337b158ac50fb76e8af2e56400269b1d6abf4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/84206e86f2ea2e53a8be26528b1c10fa5e06648d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/8f4768d1008136617a4dc8c3cbee0158a763e312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125841.92/warc/e86277b3ca249ef6236be6fd4f4f62a4d8794c61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125849.25/warc/5c3ac9a439fd585b3a55672db8b8c9a85351e51f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125849.25/warc/7a78cba8f382fcdd79347be9123c069074d7c184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125849.25/warc/b6fb1f55ee9aaa26202ef641bd299870a79393c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125849.25/warc/bc50bb8164d2ee19f6be8bc4b9247ad81ad38911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125849.25/warc/cd5cd6b63126fa21acd98457b2c6976e47387652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/071d0e186585e54c788277e97edacce5124e3f56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/47b062731ae71f5e4f3595cc71254f157a8bc80a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/505007d58353ead722c29d8e25419d881159710b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/749b3c0a837fd82b41cd0b9f8f9ebfd870e9daa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/aea04819fd2fe51c8739d563f51961edbfdd6c63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917125881.93/warc/e5fc45a3d47c0a185d01b1a1f9d48f8c8e488b6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126237.56/warc/0233615d17c997c8ad2ffd2937f7567bbd5373e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126237.56/warc/10d0d2fe5c4e2fd1ec8d84e11718ca7e82ee5cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126237.56/warc/1adc6ac77e5d9a4c87da7b6b70b4653e77bdedea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126237.56/warc/a5b4a017d00dff43f68cf0ce68fb8c379c68e734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126237.56/warc/be035c753496c34aea3643e33406800d13735aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/16989b977817c84deecd5f94d2fd78acacb26883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/2e50e401b8395307bbe8235aaee48bfa6a2fc71e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/3430f6932f55f06888dc45ac0638c24377d9f7f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/a89444e00d625923b9858c648d7f3cd8b9f82b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/b779142f6fafb6c7b3199207cee681bdb4e1f3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917126538.54/warc/b88dff55e87a3723ad962371cc50267f42c08e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.50/warc/0c3556f976b9d9b2adf351977ff9b0893cbdaf7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.50/warc/150e394d54cbf796165ae5b78307d52e0e478e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.50/warc/5f97ac3b8bdb4f08e6e708b8c31f1bc7c56a5a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.50/warc/9b38a1bbb2e01f23d9dbd20a115f37b2493785d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.50/warc/dca46e8bd7a8479b831940c06ebbb9a6d49c2a79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/4057982c5137108927121b87d297a91386e365ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/4856b77cb93c458449035a37ee5827cb4c1f756e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/4fa99677f27395d2f039b164686ce4be42565274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/4fe7ca0a237a60317b759006ad16b882b6c912c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/abe9e0ef529936094b60ee0a832784f899a1b0cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1492917127681.84/warc/cec0096e903b7adba6e9cf492795871f516c5090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/14aa2a5020e9aa181e55610a0823fccc8161f344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/174e64b0fa668f8c43a05bcf3cee71cf2c752b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/19f488dc5140452073bfe2e93570730d90eaab0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1a6b333451970c6b2573aee69b58062483a385b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/1bc9fbb7c0755e79409b4adf85f56e65ebdcbcfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/20cec85eeb886c0d909a5cfef0c3ce0554aa34d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/2119b29d949b75257456fb886f948df51dca697c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/25e06d818a8f86a1837edc3f72235ad9c7685219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/267801cc66b90ee6c6b5a820bef4b06c226b1147.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/28af8e25c2fce62ffbd6cc17334275400f4a1cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/2c6cb7ed6e05d805a3f8f9fb0bc1c3df8ee2ed40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/2ec988554944d6dd386f073bff9dd38a8f81e52c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/331f1a2aebb4d4572b04c07ee9045dfab91bafde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/39dd05f0986e470bb0eeb296e62d615478124b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/3d842809fcfb319902062ec061aa6bd11934b9f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/42bd6c94aec43a53200fdd49f779cb507b4d30c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/4e6b73db8664dc49cf51f987c569dd3a050e9e98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/4ee5abddbf6af7f14881028954cd5f9abb3d7fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/5415ef074c699faae9334466713146b37ff777d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/593a94903e0f69208e4ff5f898e1f73124e8c771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/6031fb3dcc89f9ed3b45994b351c359998f86a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/6c269d2199e5ac1df7d202598440224e2e3c3283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/739eea540e14f379d22f9565b000d1957eb598ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/75253d0f4a1d62d4337e6a3adc69869718d80dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/7c7bb440503da8e2a604bc0b1565a5cec047f9a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/7d3ef98b88986f5a2f8c11281056ab8b126b6eb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/7e6e6bde0fd64f223f5cb5d55e0815a3ac4a46d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/7eb6dec4968671bf6bdff501857b0173d1ba134b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/846b27b7f4e54374d07d57e1f077f072171a0f44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/87de6beefa432a7075da342ca594056e064bbbfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/8d0d6faa73aa1d9b98cb9ea3b49a5c23a6868f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/8e88789ab2183dac8ec6e589e4a9ca8154f8d432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/8f03b2ff0108066503d21b4de24a03acad75dd71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9073816e8649c688a1d3c97b947b15b096ca40a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/919e2ef68ad940ca073912318ed258bc9c60e1cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/92c58b8c559f442168aa7ebe7060f1352dbb99c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9412299a9359b9dde7e6f94494cbf812505fd869.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9569bad95b2047a8328ce516bccbb752d944a3ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9732fe4e9a85e662218c444893fdea207d9af984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/98b316b9040e670d4685f8985858e6ba455a7c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9b1583065823c591f5b6581c4b1870a822a18c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/9ee74fda53b8cb1dab5bba08a82e411ffb2f2c76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a04adcd8a1f3f24f67059d2b4494b883dfea49e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a15ec1427b38e56c5ac849cd4aa89dda4a01ecb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a1735c387d5c18c988e3bf77ffd964ac097b3732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a180882cc29d5212782bca1142ed0a1cb2756f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a41654bc3ce19c189d25139dec2dfbd063f73c13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a41a5c43fb4b45739808586fa45b63262d42ab48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a4de389a8ce4182ae26e033c1f1837cbc67af119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a5ab217399ace3fa353583a35fa2bff59324142e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a5bdfc704d5956dc25724e9a99f593ca3eedd1e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a708526635c153dbd1443e89ae75086e292c0aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/a96b2e3270ee7eb13b527cc89c3cfa2b243cbe6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/aca25a5176cb191dff1fafba5620152ea7bfc4b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/af6a319349959b70eec82753a22df05ac2e1f18b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/afb9315e0ba7ba8a7da72ca297e85839893bb4cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/b8bbf527816d0719723ab831b296dbe07c5f302c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bb0e98cbcec75975d1c159a743d3f0013c3b5188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bb7ff0f3b33ed25614bf6f2f5f8a67b691f7b0ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bbb3a6c2e8ef4cd7be78d2164b90e3da6778cd94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bbbf8f6158ade763cb46c0fecab2ad9af738b669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bc26f94c7cede802da33130f87ab02eff0723b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/bf6492de0948a7fcb174f28ba2cdcdf3954efd4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c37bba4e088f7fd8b372269938e3e763b35639ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c38d2cd6723f98317edc32017f438c3479a982c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c3e99075aade44ba69df1c3725f85db20da8113a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c5e56fb070461303d266116c878133ddfae74ef2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c78f6c786d2715dbc9c9692aa1d20b63651b4b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/c9eb8d848372fcf1758a448f6942105d69e749cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/cfcc16930a38f39d80c2093f067b053b4e1a4b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/d24688eb8fcd4c6e28163916c5375a5968b2151e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/d2beed8709aa181a44c2731fd5cdd01334aff937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/d89d3be5c3233f9bd82c44a8d0bc2296368a5449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/d9f05ea0687a02771d4a0687c082778dbc4e48b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/de77ba3318d66983fd8969c466f2fe01cd4bf253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/e04af5415e8ebd6a0062b8fe5b95d0723c75cd01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/e1135fc50952fcbc2e7bc3f6316a6dff164fd7eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/e2f4adc31dfafb058a1686635abd3fced07335ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/e541db0faed9443ff498fc92b05d4d8b9d0fd07c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/e6ccd85550e62295f70a0d9b58540654f4718fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/eb867234d3e48295e0f205160c4d8010324fbc5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/ef2775bb1af54fd1592d2b9c2a9982645460396b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/eff89cc2ad781ed8e0182b855aaff8369e355d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/f0c0540b55bfbecab75bb2fe9323972a5c3fe89a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/f666e50f7a0ec783357222a42460ec2595e773e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/f8eed6e2ac201c7d95a90c29f1d85b1e5a28af17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/fac51ae0d1573e21611a5a5386d9f6838c5770f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/fc311f44da6e1144cd34dcd3c74a8fafc994af13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/ff823cbd257a3d8a156a38a45ba346c2306d28ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/ffd956964a28ed3ff4732189b29798da99d3d198.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-17 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-17/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2017-17/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2017-22.yaml b/configs/cccc/mixer/CC-MAIN-2017-22.yaml new file mode 100644 index 00000000..b3deab16 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2017-22.yaml @@ -0,0 +1,682 @@ + +streams: + - name: cccc-CC-MAIN-2017-22 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/01d98e3d2760a8a499fada423142b43e8ffbe6ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/077433e899def293861c8dc7983531acd83d34f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/09ecdcbd7e01649a5f470ab965f1fc3151cc59f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/0d34f7a8199d60678ccf1012df9b0a0d7666a16c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/0ec0c1a6c64e3dd9eea4b43dab3918b4cf63acb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/111235e318d8284bbd4bbc0e142d05a3d96c17e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1197cb8cecb030b1d57a2986ca3d1b18f301e61a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/057e22712491c48eeafc682f27ba4244bff8559a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/6ead7943fb083c9d0c4c32a7a785e7d720d286b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/c0d5af31cf3a33f7786bef75085465506de8c51c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/ca36c37e119d597e63b7020e032d8155fa760ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/caa1ede541a02c006a65ec61bd65b1b75aadc004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605188.47/warc/dff6a1c24ffb1c6338dd45cc61fb31cf30bd2564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605485.49/warc/2877a3ff5646c0502d7f845f6ea7b3fe0b305adb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605485.49/warc/5cd9499b011aad5ed8c62cc9c8121df4ece2d54f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605485.49/warc/a80027c42e1bd677263b8084f5dd8bdf5c042aae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605485.49/warc/afed94bfcecc742df8a0a40204ecf14688089f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463605485.49/warc/fbe933edda0396823e8e9fefd6a863bacc083b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/34dcb0eecc201f0bedea7b6a7bee8a938ca60ad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/48467a64582ed5426641e0338ede6a72efd42827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/8a69e4b0335bde274983684fc1a64a601062a073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/9fe74eaa59210faba0db436a72049dd670561483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/a5da0ef3a1d6f29d1bf12f764c4e5d7f77bd7306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607046.17/warc/ef213702e6a20f45253ce1702d047869786c742f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607120.76/warc/0a901ec287034fe4fb3d86c4ee809bd7bbc8888d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607120.76/warc/157ce22b8f69f31c7b24ceb1cdc6928f64d7de05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607120.76/warc/25d1991ef2ae196aecb212704fd9da4b44e8ccc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607120.76/warc/cc35b7809ae170ed196bbc0f76062863893041d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607120.76/warc/fac28cebb09fe5985b57e14267635c92988cd9aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607242.32/warc/109695eebc20988ccecc88995f89d1e3a06510ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607242.32/warc/351f42d93b00109fbf3045f76581207f479098af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607242.32/warc/723b5c20f245173b46ec313fc6602ed659c6b373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607242.32/warc/a715cede1938125abb1bdf1d16121776eb3b0f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607242.32/warc/d4894b3dbfca401d40ac5761f5e72f5ffa5927d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/499dc89969348cf3bdb290a66ba4e4b722b80b8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/76d2a6e943896cb7231f44ae488559c86d6502c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/8fd1903b58271e6ba45ae9ed9accd6fd91f00166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/c8aa4d693375c193a328f64883d979c9c3eb440b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/eb74b4aecfaad5b2ff9941e46dab2cf1a15c7b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607245.69/warc/ffef207b5c2927fb39717467006226c9b29a7193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607325.40/warc/07882e82c07aa5161a611c71ff31eb44ca390e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607325.40/warc/85684574f99e1e0025cf4f115f8bdf051ce69241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607325.40/warc/c39632a16f8a53780b7d555673164672e5f61197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607325.40/warc/c43154850be7f93073263ba264b9a05ba0b5e299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607325.40/warc/c6db036213de305afb49d03fe1a656cc7ac47a6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/3b327fd721dfae27c0f6403f272eb5397a552a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/75e931786eee3e78e74877f01e91c755152f20c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/90d70e728862f1663ad705376fe54c440407e929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/d9c51c7e8931af6a6c43350a7610f3aefc0384be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/e1601810cfa18e5acf1d0b64edf240d16cc41cfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607369.90/warc/fe5c636802fbbdfb6427a0bcddd4bbc78019d04d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607591.53/warc/06db369d413f62a2413d97942166ddddd44b0dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607591.53/warc/264b2720b13d356be9b7dfd535a1da66e5de47e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607591.53/warc/7483dde979e1b99387bbcb5c75c95f1d0d7cd3b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607591.53/warc/86070f00b25aecce65f848461e6063444a8381f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607591.53/warc/f82c241cf0afa4cb0e67fd6477d032f37c64d249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607593.1/warc/12ddacf3830aa4a8ee26d68e5549005d89640b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607593.1/warc/c7630853674de2c40572c6ba8c442e4ad379cbfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607593.1/warc/dc1218c9b42e7254e970f7f388a71c877d196d3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607593.1/warc/dcaf01f550734dca421d7dfde630f1bc3472062e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607593.1/warc/e721c94c77bb6659e516f39c603b986cac132792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/0143f6534f2831f35cebcd74e8d6279105e43048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/1b94d19df81d7f28ec482faf0abb864196e6b50f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/96c1fac32622da83b05ace8f188c270ce6ac19ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/aef45dd6586d6f844cd63a63d8f52513976693e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/b3b4f0e303601675709ce53f0ec21cefecef56ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607620.78/warc/ccc469e3dac249b9d2f511e6844c9a5b2b61d69e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607636.66/warc/5eebb98665edcf8c0212cb5cde177f55e5c01ca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607636.66/warc/63f796782512c2c115ae12def3a6841be7c1e5d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607636.66/warc/a5d2a738fc4d0cf751992695c6bc3aa5ee7a5c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607636.66/warc/b6a6bc20e0e99116d8a1b424da5c02647b652568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607636.66/warc/db6e4d2c3693cbcbbe1d1449ef22cd66d5c8c9f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/438d54feea96e3b2a71f8ceb6d832841656372de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/70d9370402dab77e201c6ecfaf35be59cbf9e0eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/a09cf24c1fa8a7796317ebc66d2f1919a255166d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/acb506328d963d9c3f8319c616a482fb7c452faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/b07429d34fbc564a84b0024b560c16663a5f2248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607647.16/warc/ecd0cbd77e7b4cc5f71a0a1a9e7a3e1026cba486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607648.39/warc/1a5e50b1415c1dd235e290a324913ebd24ac65c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607648.39/warc/332581c816b7f1d6d4661549c3dfcb6a3ea5ad28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607648.39/warc/9a1f3ebd634de430d3f668ca950a50927044c97c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607648.39/warc/c0d1415f88c910a16978d56e771bc4cbe475883f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607648.39/warc/f797eaf8e29df0331910e816fe8db5aa0a7570b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607649.26/warc/17e974a0050d2cb52cc8ed300540eea851f1a0fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607649.26/warc/5a0bb0bc3e1ec71db74f5cb1c5ad65700beacd56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607649.26/warc/8eed52edea3caeb938eb9b4d37e5bcdb72960b4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607649.26/warc/e2c735979f52288ce9d7400ced46c81f71015fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607649.26/warc/fa42a0d14606f37193c02eda227b6ecbe06f5358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/357fa6779bb0bac537ab89693466c192c1f04f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/4fa0f479449a3e46da86c45bbf995a75036a6b9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/73d75c48fac154842bb8658ad6d7967605da7c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/7f8e4b340c2314837fd3cdc37f2db8dcc8f43968.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/aef3066f16812b1c8797d39bdda6587062ac6e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607702.58/warc/f1313f46ca43561942296bc16ec240887f3ffad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607704.68/warc/55033f2781a40c3b57403a31f4ec7d72954f7735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607704.68/warc/56894bce5d96ac4878563cc280716a97ae2405e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607704.68/warc/5b25b1b28c8ba3146c3f0c7de8fc9fc4dae8744d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607704.68/warc/9cef2a6d6c750eabf596bbd90f95839c98ccc96a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607704.68/warc/b020431246ab85f9e21804b23535bdea90b1af94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/1354ae16dbe67b122b4315a6ea309b45f0c94ca0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/94843bc620086339cd11f5b9ab72613ff5300b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/9f985022722ae542e5c73ef2620672d96d903795.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/ea7464282387fd4a5e33e39c342e2f441d259af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/ed335cd4994d7d7a2c68c68b5e4fde50f7bee08b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607726.24/warc/eec21c1892b722bf7c903564adcb94b5256796ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607731.0/warc/0bd7205fd5131bdf4128f68a69a8244e927b780b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607731.0/warc/2a844dee7b6b945a4e872119b1cbcafe2cf00223.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607731.0/warc/6756f1898ef0f9476502d17ff4ddc57b42245cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607731.0/warc/695d4e14013bc4860bbf0cb49d9348bc9596a12e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607731.0/warc/d975ae7bba37714f409d71dc1b201bf70098c135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607786.59/warc/347ea62e840ca8f7d3fbe133736a55bdc1e3c55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607786.59/warc/407dab28281042624dbb4ed68bcf03023b71c387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607786.59/warc/5adf764fb830fc157882e165528a08c58fd51b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607786.59/warc/6c3973e93c476b0dcafeee629e8522670230a43f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607786.59/warc/a05a2e01310742f9a803d6cc0963825817c632c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/85b00e79d4aa41dc10a9f0cfbb97b67641671921.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/8c0d2481d7b96e3e3bb0d6891da2cf6dddc4a6df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/908fd56194379fa7fc11feae0fed94a694283e7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/b2ec077988a324fb353c6736ded1d0aee9cbf44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/beae0c8ae478efa936429e43faea6655254a7ece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607802.75/warc/c1d66eaed2f39ae932d2ab82024995a9f4883b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607806.46/warc/25b4bf747df1c24548b91d9f9b668806db6afc8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607806.46/warc/2cb6f2bba73275261debe859b9831fa125f3869a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607806.46/warc/3f7c9184efa3c8b6a95a80e97a5df711f55827f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607806.46/warc/98f6a2ee39ce2eb96c1908de8709059228866c61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607806.46/warc/f28636aff7aa0faea93ad5213c840f53bae1be73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607811.15/warc/35528e156131b94cc535a60346f57c2a35dc9e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607811.15/warc/3956ae60640bfb2819a3747d2cc9b28d6cc07bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607811.15/warc/841251a06a902917afa0437899c1fcff9c454d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607811.15/warc/d7931d5cdf1865ed8e0c9d281554a88befe69f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607811.15/warc/e497ae2a158a0102793db812d5083ab33a333ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/043a16e29dae8492ac8ea75352148ed8186dfb1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/2be26134652c38c3190bcc67adfc6e058830eaf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/362b88ce038d044d06de33a5e54d89d0cd4fa984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/38af72a8ad8d24c6d9c3c60532c30e84552c336d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/92586b5389206a8d77fcbd5db78650d4d3edc020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607813.12/warc/d8786c39957b9ffa3876e9e6f1d212452caaf6f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607846.35/warc/14dae823719bd38793adc6f596c6eecc3800def8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607846.35/warc/1c72ea0d724e5337046e4c52c1ded1767d6a71ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607846.35/warc/5a809ea814eb2327d6fc8279e6fe17c6b16377e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607846.35/warc/708a6004da4d995ae352e74e0b4fbea23c7e6b09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607846.35/warc/f4f0139af8fe467b88a12463390498be95a36d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/21e31f22a0cd58fbbd8a6f0361c6966d811d2bba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/5285d9d3110a38705e7f2808ef0e2cd4941cf018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/baf4e125cc7bd8f42ba1824c679010ca50c3bfea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/c89cea32d7dc9449e720615d6d9af3791bc45138.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/e6be232aee81bb670bc123514be1e0f2522607ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607848.9/warc/f8b8c57e880bbc8fa1d21215943b666ea32dce3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607849.21/warc/1ec5bc346176146055a6f8b145f440d3ebb5328c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607849.21/warc/34c2c2a32fd92b04a0b39cf6d706b7a2998cd671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607849.21/warc/3d50221859546db6b6b28245050ac8d8361d432a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607849.21/warc/8f61168ea2395bcb88bb3927fc40e3604199bf1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607849.21/warc/ec307db757b25e61e8e58c7e1da4529ef7eabe37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607860.7/warc/03bee945db9691eddaeb750f855194587dd162e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607860.7/warc/276b0586394c3514b792bb462fd3e7477f013c96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607860.7/warc/63f4958f7d8efd7a0edf2ae4db1a559caf36ad7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607860.7/warc/7bb24950a15a926852b87ee2adb2e69a6a3a30c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607860.7/warc/c652ddd84ef8b5f94884efaf885bddc61bcca01c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/0cdf02f7f1d6bee966c4e8e35f58a0eb71a4febc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/8c141fc602af960bfe3cf8976da197c3724d8abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/c653c8dc7da18cad700c57ff259c2fc59562fe2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/cc3cd7753268110f30338f37e8f727686bd2aa3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/d29a64d4f5ce332fddf63512f3f32e849c7cf3c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607862.71/warc/fcfb042f0e2b9b5239e19f0e5ed96e558237ed43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607871.54/warc/36070419297551f15c1da1776258f5fff89c8a2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607871.54/warc/4b534ac1fa757d68f257a0ff1227263d07077ef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607871.54/warc/82e4f5b57e82dbe955cb7650a8cb2d502782e5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607871.54/warc/ce97eaa6c45364657df11c2b78a75bff14e70a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607871.54/warc/e0cfe347a3b7e48cb07f7bfe4b5432ebf85c9643.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/1b3493ec15cb9f445ad7f0a4ad58de36d242922e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/39d5d6488146a688ec4588513fa6167e45a720d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/5b9b54c9610a3619e588a7c8723c8df0ddcde84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/be0ca581e6cf65f5207bf95388a36457b91d21cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/edd47175f3717e72c41df6cd62367596e67f21d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607960.64/warc/f0e71636fcda762267138efa85b776de8534f881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607963.70/warc/4155a34744d1c768b6e5e97ecd3471fe4ed0e015.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607963.70/warc/4826f9fe49dca1022896eb5aa5d6be5140423132.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607963.70/warc/4924fed0bdab60d528713c5e9b880e791b954fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607963.70/warc/971950f9ec8f88eecda03d930f44e7ebdf49c7fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607963.70/warc/c3eceb0fd7a8405f9b8886014dc06bfcce2f41b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607998.27/warc/33e80f1a68f3b2973203cb5a5ad33610f5591586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607998.27/warc/6857b3ee35fdf3b1b61e2bf1a59b007b51e17f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607998.27/warc/7154e6df6537333ab3c667b3f8298c1bb027aa60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607998.27/warc/8be92c5aa1fa1dac631df693f7ac49626c72a450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463607998.27/warc/bb9c0613aa2fdef0cb8d626191e1ea6d06151aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/4ab258b443b562c784cb5774e3d2f231ae7d78d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/67638f7425b85668a12b3315f9e9f6a3810087d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/7e29f8450811e95d23f7743afcbda37d65e5c675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/acecc14a5766693ef5c747b9e95bb16e247e1bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/cdf48e8d0a0dcb80cc98d7cac3710b8f014ab55b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608004.38/warc/d1df63110a0565192c543a3df415ba5b24e92911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.13/warc/7b5a15ee6f12d924797d3ccd1d05a69c7c02840a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.13/warc/9ab59c18ee01a486dd1ca5f665d9562778c8b34e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.13/warc/9c213806ba8d1c4b248e55655f8f2608465c9c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.13/warc/b3cca513fa2ff2469c7479c55b78e1898edbf2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.13/warc/e832ae494b066fe5cb2e77fd3d80de65999618b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/54f183e9c18d9f1f6c70e1fc3b0a051762601069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/5e7745efbe42bf825a88c3770cc45b404b130a8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/8fd3643f42daaca700b6e281aa71b1bb2aa63e6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/9b7df62225498cbd723d3b7b0482d6acf3883633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/ae55e4323522dd1ba9910c5575abf8e6bb295978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608058.57/warc/da9c595a661c9556cf7cc42990abd8d81627fd9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608067.23/warc/086496816ab8ff04e1a34c5c75f39328e68f5a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608067.23/warc/54c6e29e48ead3a77ffb928901774cc8826d32d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608067.23/warc/6b53281ac7be8e3c838847bd18efa356a6752255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608067.23/warc/74d31c0e4098b2ec030a0541e8b778ad5c702acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608067.23/warc/953e680b16bd6f2c11edb9f7ae3653f19c118012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608084.63/warc/13b9be15f88f901e937309db610e2ae828898bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608084.63/warc/37e8bf8ddabd513f628c7f5d6cde312cb5a870af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608084.63/warc/7c5980dfce187157a728f890d56ab7e24b74375c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608084.63/warc/8837c8707de4bb19931c323ec84f0a821d3aeb45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608084.63/warc/ac269b23167976417693b6042560e085cdac20e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/29b336d66088cd94a97c29f6097759e7c9ab7323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/59fb16987d3d0065c6db788a8cb4a905dcc27cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/7899c8d449522fd999089faeb38177ce54cfb14e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/8f82a3fcd3448efe14d18ea341d32db91cb88de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/b7b720ff53627eaa14a6517da555bffc64c1f2ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608107.28/warc/d08f6c80920111166c4aa3ca5f5f56a847d01222.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608120.92/warc/1b4b608265a1b8e4d923c06248b9f800d29667de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608120.92/warc/7e245685e2e2a9902c2c9689b96cb1bddc19c4a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608120.92/warc/93929aec17ede6b23161936815c4acb7fbb1c186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608120.92/warc/edbfad37408e044b8c8941c765e572a3aa40b112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608120.92/warc/fb8187296996852b3c674f7682d1170b631617a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608416.96/warc/0e60652d7efb35e5e56660f267d5a6e8bf663969.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608416.96/warc/34cd2f5c9c5b65bdcdcd93742380f10ebb89f781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608416.96/warc/7a8e8ef8c352ecc44437d5d867ade14e6e69a733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608416.96/warc/81cafe46d02ca2c72874e2ee6a9f7eeca4d78b8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608416.96/warc/c2b3fe28e6625b1828f9b1192c162664ea8a0f91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/2b8941d3754b363a416ed851023b1eb1967ddfd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/36a6f33e5f6445f1d9f9935dd0e988d72397eeb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/8773411c5aa261d8f7c4b8b50b99a47f46a8fd53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/879167dae1288e248463ffa1a651d73876264715.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/ab9c5cd039c9fdc4604e970f049fb8d406322f75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.6/warc/abf1bacc87639490f86a8431f55817aac7629524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.80/warc/3e08568723eb349cbfc40030ce37a2d8477300f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.80/warc/4b1067b4fa1d45337245ae807f483e2523fbddce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.80/warc/de8147b6a019b9565cd5b0a807faed15bc740ad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.80/warc/f630706e20f382979c1bed1f78623f6380369b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608617.80/warc/fb04673a5b6abe888fcb8d7f48285ef9b0ba3e7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/13bdba20679599ad9b58ea912f659ba8ae3c27a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/273107ef03a9e9a24b26266c0fd2f59ad0de6990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/28c8d04925e91b1fd64da53ecdb0bd81b7461f39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/8f66648e02db3bb08911a1d6af4de84459f3d4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/e5b04c823b79b4ac4ee84dc9a1ebee19b39809ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608622.82/warc/fd5392876c7ef8429a6788c93e03a1b55ce8ea84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608633.44/warc/81f07d95c05b8c20a14aafa64246bcda0ac32484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608633.44/warc/cac6a989a8ed4e8abb49fe30bde98df04b6df11d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608633.44/warc/d9d8839a8931086929c4c2ed9bfd0f30d05b65d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608633.44/warc/e672e6f916a5d0412a7cd971c5722c4158d1e0d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608642.30/warc/20039d1bcfea1541b05716de2ec41eead269fd77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608642.30/warc/6a0ecf07d219e660340e0d7a735f9a58e3dfb2e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608642.30/warc/c4da8a23ad53a891f84eb4cdc733fb07bfaf41d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608642.30/warc/eeed161e8b952f8c8cc0c0870230e9cc1ac0995c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608648.25/warc/653f8a88e76bdeb31070a29a0f9cdfc6d3e2ccee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608648.25/warc/8354100b17c95e4127c19f3d9b83cf4797b0d42d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608648.25/warc/8442976d69ff7d093eca25516d0a2ef8174083a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608648.25/warc/9df41e5aceb124f1a20e8aff2ba10f6cbd767d32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608652.65/warc/606f330d564abf83629424b9e6fd300b8ba40a83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608652.65/warc/c269899afb08f7a7d9b6872f3054ce6eeecb86c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608652.65/warc/d8eaf1a01c8251542b1ae9aad146db1933f46166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608652.65/warc/e5df6b4d3d9ca3ec7722632164536b3ecaea77ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608659.43/warc/11b1373a158f806361502ff82ea56aacc06052fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608659.43/warc/216096d612d1fab967ab07d34b1cea8c37d12582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608659.43/warc/9f1000e46ac3ad444465cea09ddde6bb6076d93a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608659.43/warc/dbd84985f8901ba14a18e79e85796c73882f3cb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608665.33/warc/0e0ecada9a4aedf7112c0c24c0a9a52dd999af04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608665.33/warc/27b458a6ba1be7b7badaf96ff28e796f0f797bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608665.33/warc/293a629cdfc8efda7068a5b87e838d848f7e1462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608665.33/warc/7d3ac185586dac51769acf7a86ab42951ac31e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608668.51/warc/0c7aec84e41cb4e89a5d4767e67753b25c83684f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608668.51/warc/2cca2627415b64d7f2cdb739a1171edd2ff13bee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608668.51/warc/370707ca5fc834eb24baa0f153afda7d1874cca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608668.51/warc/58bbc570fa4b140e75e24ebc14828817b81ddae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608668.51/warc/bccc04a99eb5140e25c651ab41bd11275f129da4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608669.50/warc/2c8b73f6d29ea83714d7718ba256ef7025133469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608669.50/warc/8cbf56634507c3bfc813291f922edc38b0f14273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608669.50/warc/9099268f09b3b344575fd7377c2301837bc8c810.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608669.50/warc/b5f718b8c81e331e4bcf8e355fa8ca9df177bf59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608669.50/warc/d46f29e632e081061ded1c74feab8a66b770761f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608676.72/warc/0283a8a6197434d6e459d471504f30a334af8b38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608676.72/warc/3756d697518c02bf6b8ec5c87e2acdb8b7dc18cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608676.72/warc/3eeef94074f0fb621f8e0bec8d4f0fa92b400984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608676.72/warc/68ed922b0b2573e0261f2bd27aac6722d57770be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608684.93/warc/0d5d580bf67d7c93198c1182421b74398b733d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608684.93/warc/2cd6e1040f21369732fca95386d71b31adf9816a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608684.93/warc/3c7e5d68182c8fc0a2734c3151d94f21ad998e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608684.93/warc/f111ae13b982698be68e934756a65b7a99b20ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608686.22/warc/65bb969e46a40ce91222e27117a286554dc81044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608686.22/warc/a26538611cc8b03c94018420ce11678dd43f5928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608686.22/warc/ba34ee18e8039e0e78a24f46cad727ad06b78761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608686.22/warc/e35280f0e380e80b2debbfb9d8b3c6796299259c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608726.4/warc/50bae75d24e67198b824b439b4b50eea51baa61f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608726.4/warc/5e323a31d8e079ecc4b27097f29fc6df2eb056c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608726.4/warc/f92c5dc8d9998babe9bc0e0267b3375cb3bd0817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608726.4/warc/fe61d2deb865e323497fe10b8729cae3a2c99c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608765.79/warc/0f5be8eaae4989c3049131c1901e39682a937238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608765.79/warc/13a3d0ee4f7e3f8002288761e16bf07cc8252d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608765.79/warc/405dfd044db85d59b42b9227ab0199230b9d03b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608765.79/warc/e78c58d4e5be0393abdd44b0a47647055ea835fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608773.42/warc/459310f159527d51fee21de495ce311170c103ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608773.42/warc/93b886bbbf17e41132f2e54214f2b2ec78bc7454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608773.42/warc/f3bcb8d244d74098f8fe05bd68400a6668965371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608773.42/warc/f6bb562f730b697aa61179c3592d9fbfe492c712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608870.18/warc/162d8551563ae934dac14a62c72f663032b54aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608870.18/warc/25130c1b516b82381729db39e8a61c83eada7032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608870.18/warc/7b4870360a4b86da714dc1537120aa4df892ba2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608870.18/warc/879cb8dfb0c964eb3fb701171bf6a8257193b43a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608877.60/warc/58209db6f81c691a260576dcb5749140aaca1a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608877.60/warc/5ff1454249c5b5b657c040f997f23d77b98c6a19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608877.60/warc/9f9c74f0b5bac7bd8750ec592f00186c0717d790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608877.60/warc/acfe8b693673342442feae353c660ae323dd85da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608877.60/warc/ca91f318f4f370ecb3250bc61a88f476391233ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608936.94/warc/33d3dc803687097d54da9bb7c1b9b107ae96ea6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608936.94/warc/60235ef7b1a131f4c9a24f541ede6a4bf05b4011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608936.94/warc/9e3bd4274db800154bca0fe353fa611ad7e79d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608936.94/warc/afe01064bbfc13aeb6ba3176d97b00d8e0b6e5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608953.88/warc/83c72f390f36f4fa274695854f1789e238f6f41b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608953.88/warc/a4dc24de7ce8e98f65ed993321397375cd5f0ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608953.88/warc/a6429ad6c1d190e3686cb95a4396626f344e3a33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608953.88/warc/b40d0fb5ddf02ffb418d2b2a87e42e4cd6404622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608954.74/warc/21b99a61339b77588f6beab241e185b820391149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608954.74/warc/57860bd5d3220a3a626ee1ddc3ac194269ddeef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608954.74/warc/7a876e98be0068287ce9ab2ecab2631ab1eba004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608954.74/warc/f6f3d15b4e49d6c84362b0a3dd8e2a597340760b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608956.34/warc/2b8a3a0d6d825e2345fedb332dc467fbfb9ffde6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608956.34/warc/325d45b34a6ad3c70235d25285be63cd1d24df1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608956.34/warc/78fd4bf38a308c42d081a51361bb7ee1a27140a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608956.34/warc/dec4566c6c6b06bbb67bc374de3fdc04d3adc897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608984.81/warc/1a7949dacde817c3d3341297fdc7f2b410ff2bd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608984.81/warc/1f6017c4e13e2ad4d8eedb47ca3f40c9348b3d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608984.81/warc/6f35cd58e9f719bcd6d8aa744efe8e83c488ab99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463608984.81/warc/83535ac8f2147cb145ba27aef71099725e2a49dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609054.55/warc/369b74cc54318015c3dc682113d44581a9a0fbfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609054.55/warc/91a238bf1619715b0c621f6e8ec96f0214293c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609054.55/warc/bc7d52694f49e9de7586244b269e58100787d89c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609054.55/warc/e53a34c5de1e3182ab7401efbf85d887735ea50d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609061.61/warc/05d4803640dee7445c9c72857969db3090392286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609061.61/warc/47e7e61fb9b70374157dd4cc8811ee0eedc0cbb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609061.61/warc/c956b180ca2d8d7535df6364b47eb229ddf8d417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609061.61/warc/d74e918457db3373bd4f6d9c8510b2a523842cd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609305.10/warc/3a69f825b63c88c55e1d1e63d7fe3b526ca1a312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609305.10/warc/4db48f538acc4a2bdced6106f50d7f273db94a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609305.10/warc/5e94dad7f5c5b438aafcc4054249cfe3cac19961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609305.10/warc/68551b3d3119c53bbd56decd2ea8ec39cb3e275f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609305.10/warc/69c9d80bff93b647996c14d52a0bfea856ff180e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609404.11/warc/15ca386bf178b2d2512b222812d6629f75360e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609404.11/warc/2ebd5327cf7412b175cfdff2b68062edac9ac409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609404.11/warc/9e870c94d44325764733e88eb490bc4151e6fd65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609404.11/warc/9f039ac6d95335132bc738851d8f0bb7787e3e31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609409.62/warc/3723a8567709d68e35de0039b090edded22a347f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609409.62/warc/8984d0ed9e3fc143e6810d6782cb9161c53e3140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609409.62/warc/ce139fd202cc1e18c84cb4e1a8695c74e74675d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609409.62/warc/e6f6ab5ab0a18dbb8f4789433c2b22f255594541.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609598.5/warc/141ed2b069da03e0d5b326b20a246339d4fe232c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609598.5/warc/b444c35f25db4033a7457c6332d4715f86fdcf21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609598.5/warc/bdfc22b2092d8f90f0ad34e6a00c357222ff31c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609598.5/warc/c1efa7eca610baefe6f65c9139b7099cc32cd04a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609605.31/warc/3b2c5974dc1c466855ecdc753d47da4f1aacb62a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609605.31/warc/701999d40dfbaa2a79cdf28f463e6846639ee406.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609605.31/warc/84123db2e28abb827d176fc7a2d6612d6fde43bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609605.31/warc/af79ff3265892978d39c8daaaf842e71080db761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609610.87/warc/19cd0c27d91a08383a749118102724161e5fff34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609610.87/warc/c1122c0f3d209c9e2cd626fc98177008a46b65a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609610.87/warc/ddc5443b777ea4c8e83edff5a57f4c4b700bcf84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609610.87/warc/f90a56b50f7f5ab2d3eb2369f5f86a058c00496d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609613.73/warc/4373696d2f0d289e6cd3079c5c6f7e5700ba05e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609613.73/warc/5ffe14b393bc904f1bde84695c8574e0012f029a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609613.73/warc/8cde3c61f380f62d720801cf60962d50e9417ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609613.73/warc/c95feaf298c0ebb105c53c3e2bde2610292233e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609817.29/warc/3a37ee94974df751f7e9f040122e6c7c8dc49d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609817.29/warc/866473afa947940ac1bc256625d9a935a2591917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609817.29/warc/ec05704866692d00846786829541feac6db26d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609817.29/warc/f68bb2f35712ed2a21329f7c4d69215864f8fe9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609837.56/warc/2f3a534a93d276ca8fa201c44f3ce527357f1e8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609837.56/warc/b4eb044ca991b6dc54fa74cbdaf31423174e3b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609837.56/warc/c47d1d4ed36bf7e86068f533f76d4b48dc048013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609837.56/warc/dd3aeffede2715c31cb0433c3f44d4304460758b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463609837.56/warc/ef1aaf0e880e1586adf607726a3253b38c734b02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610342.84/warc/6da7aeeaefb78ceaf60635a8384607b3ca370955.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610342.84/warc/826fce03be3fa4285a08fa3273950002ddad6851.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610342.84/warc/8821d7fb37afd01056f23e253227cb05bb730b17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610342.84/warc/a7f003e0fc0f8df9cb0e717396b5561194378d30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610342.84/warc/c7c4549e2a16f7c1e10bb1400ba5f41bb0713ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610374.3/warc/17ba68c6f17057ff4e03b7abc586e726b1855731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610374.3/warc/319aa65795cf69728e5720be7a7289c7fbbdab8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610374.3/warc/b4e63beea54ae4f7ce3484b5d1ed53dcf1b5a910.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463610374.3/warc/b562af5c0b9d0f49711c0df9ae23d740f6957c6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611560.41/warc/08a02f69513f0dc5de65322f0ed11620cd0355a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611560.41/warc/3dab29659d3550a44128184c2c1efb3f017ef75a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611560.41/warc/6188c3e41522afc2f9247ccdfcb27e4af6c166b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611560.41/warc/7e51a3edcd80ce5ee0f645e4fe2b6c8aca5e9eeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611569.86/warc/05fa973abc0b9c005a05f07ae59d861465b76196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611569.86/warc/5cf70bca5b278b36d71931888dc2d98e1da4b655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611569.86/warc/7dd874f205b7667af9551b155c484dcacfac47d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463611569.86/warc/a869f79ad51e6d26113d4b3c25c1ac8c041042e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612003.36/warc/871236220668395ef5b029ffd13f9631c07af047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612003.36/warc/bab4961848c873149afd53bf06ce1de1e42cad04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612003.36/warc/c51e032e46b7506f7b67f6a19c76aeb5f7ffa3df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612003.36/warc/ceb58a02fabe76956fde72770760f89272181aee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612008.48/warc/5eceabef587df2820e003a4862a2d0f5d0501425.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612008.48/warc/741a4a1372e9b21b052ee507c2722cf330c9dee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612008.48/warc/b7a612b0069e83e31bdea79097d114058e89e007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612008.48/warc/f54db7bf6a00bfe01c1a7e84feeb6cd7fe3a1cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612013.52/warc/770d18c6b1801888ec3cca661dd668c01850b5a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612013.52/warc/8a45d5d40f7759a466bed587337205b308bdbc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612013.52/warc/aa02f2c520d5860bfa8cb0b78b8440c7083eb591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612013.52/warc/b44574c931bb52b02f59a948ba292a02227637bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612018.97/warc/4d8d01de3be2a952c1143c2b78fc8c950132c228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612018.97/warc/6059ba218435ace319a21f7ac87cbade56df5baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612018.97/warc/adab83ca78a071f8017f0837bd7e454edb5c0165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612018.97/warc/ef243fda1f28e764be5c23b674245e4e8b28e1f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612036.99/warc/05592b010e06929b8fa1be87ba34ca693c071903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612036.99/warc/602235320571a6e14d5aec0d418d3b00b22bb2cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612036.99/warc/6ae638b54490dc2ab24f6cb70cbf0d0f239fb012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612036.99/warc/a543f3b1bdb9b28b75c9abfe252b45dbe853d172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612036.99/warc/cf00daf89dbf0aad4e737beb0f4eb30c4213c7cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612069.19/warc/0316ea960f21fde4a452a3cd2e0cec4448679022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612069.19/warc/93005d02c05b0282d15dffad1388082bff45468d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612069.19/warc/96467a611551bf61a6f6011c707cf03e7329dbfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612069.19/warc/bbf26c776f1789482f357111a2acee4387d32c51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612283.85/warc/077d10f5540aec470aeefe063f60ceda2dbf8d4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612283.85/warc/77ce516eecd00a521deb5ed9126f91e7d6b12b7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612283.85/warc/909d108ccefa40725614a398b41f9fdc6bc514cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612283.85/warc/cd75f9e59439671bfcb5fd92dc561e42118b4155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612327.8/warc/07d80beab3688ed7e81c274d66e462808070c4c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612327.8/warc/456b621def9a0f6bda32dc2cf2e3e009e516d6b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612327.8/warc/713f55f931c1898b6657a4e581ce7b08f5c1fca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612327.8/warc/ab2578f99a0cd121cad05406a5c023f201ec12a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612399.20/warc/03ab650d680fac7263b129b97d48c70b4464c9dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612399.20/warc/0b6f47635c90f46aff446f35f14e3bb6f6305b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612399.20/warc/11ef56dd25c54eee525a4c8ba89129d4ff1f4a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612399.20/warc/5dcea0fda2d2d8f3caeac4ee16ccdc60dfb21b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612502.45/warc/28d39033adf06acfa970dd675cee76ed5764aad8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612502.45/warc/4295a5cafb0a6718680298a5999e3fbff17f7097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612502.45/warc/dbe62a1dc265b07746a40fde506d04ab6310bb77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612502.45/warc/e233f82328dd40fac5cfd5e66580ce8aea17c01b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612537.91/warc/704e31eaba049c32779c4e8d3e7bb36e3809c989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612537.91/warc/7b4023f41ab796298f4bdc7fff27a138172efabd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612537.91/warc/82928c751e2d45b882dca330916e114bd109c878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612537.91/warc/9d845e8550d3100fbae018488915ca5480793d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612553.95/warc/2b0fdd5de6fbb6629725054d1dbf7de4db682f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612553.95/warc/5989e17d73e65139bb27050279453352de467a1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612553.95/warc/80a3c6d25d0e1de0f83336333df508c9800eeeba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463612553.95/warc/c13de9c39a2e1382573c4b9a95ada5a14150fae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613135.2/warc/114355a33c0e0c39785b5734d40965c777a4b4bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613135.2/warc/70013dce351fdb03c3f0399da6dfb454346fc164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613135.2/warc/772dbf48f94bb9f21633f2ae523a4268ef6c2b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613135.2/warc/f7694c4b40bfeaa9a28ef389c4c054bd69539103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613135.2/warc/f868f2e2fc5ae5338280bbb7ee3eaa8dfb3001de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613738.67/warc/6a9a1ca14df9ea6f48be6d8ea5f4ee5918c11c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613738.67/warc/70789e4fe98cdb6b2ac9e212c4453785681544d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613738.67/warc/835d618d2b39386764e91fb24429aab3612a1b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613738.67/warc/8d731d17b1c78099d101c7fa9dd7295280352b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613780.89/warc/141431ab62f6edef3bb4f17d3b004a8d79c407a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613780.89/warc/1aaf459c5482362c45ed1bde65a1615ac73ebc8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613780.89/warc/d91b411d56a84491a7aa63f73ec4b76be22079a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613780.89/warc/f7306a5d36a128df36e03d19bb251a6af6462850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613796.37/warc/2586cded0ee7d5e1f154b75c68cceba33db2cf37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613796.37/warc/3cfcebc7f9caec03381573911064112e79e49c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613796.37/warc/be782c9fa035a27a815afeacb643ae7ee2241bc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463613796.37/warc/c190b5a375df3e51a8064f3ab2b2cee58c69a6ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614615.14/warc/73efca1067315c8e51cf11cab7446894ccf7c543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614615.14/warc/988c6d4b545ac4730855b6d375e1fc0af2340a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614615.14/warc/9c5d1a4c4e67cdab75103a1926d712f4f450afc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614615.14/warc/ef0b63bd95e9f7b4c8b50c74ad7dc30028a6c8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614620.98/warc/01b92c50105eedf3ef48e32dc5fa74f05808360f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614620.98/warc/457f87b40c6e8c97568611d088a55aff6e9ea400.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614620.98/warc/65fc3ded6d2af1873e44850e0bffb506ca2d3930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463614620.98/warc/b1c5dddb5e6e99650928a00e43542c91181bf3b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615093.77/warc/3d1fb0a8f6b04513b74c411220458e6a88ac68d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615093.77/warc/489466097922bee4e628bb6ddae2f0e37c35c56f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615093.77/warc/55668c3023baa75460155dbb8d2d87e849aecb8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615093.77/warc/c85af6ad70f0c9d013bf0a2d243022addded5438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615105.83/warc/086911ae8e9c3b0b087076222bde5658cb412529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615105.83/warc/16ad54164753e3b1fac01ea96a80e6cfa8c527be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615105.83/warc/ae7c8fc3a5a5b909e40871ed62e4aee383a19566.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615105.83/warc/c967b74e35b5650337a3a7f885c656672ec344c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1495463615105.83/warc/e08e9fb321cc1ed11f3ad40fa29a53d578b3ea5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/16551352dfc578e326609be96f6aa42c92cbc6db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/187580c022d7461fe08a035123ea164e8fcb2654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1941c3c17d9b607ed4be2e76895f3b17089af707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1ce26a79a267e571b3b019d477d394a4a9e96f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/1e57c07bf53710dc43f7b40224196802e24c648d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/20cc394fffbea3987a1ea8ac66dcb7ec8b1893a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/28a1f346207071e4b243a9d2751cd1eb35c9caf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/2bcd9b5a8014e98f5ad25a1cb238ea0e6df2ea43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/2cc26110eb2c9fd225ce52d376984499738050b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/2e56f08ac34a29a24926f6c33ade122e268c7554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/3299fb5db8ee39e0916487e79e48909050799de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/39ba4c8eeea16f55a3a4420e2f53e90ec98a7395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/3b68735990d29de0e32103853d26c03e3d77f819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/3bf1ee434dd4b8f641c9a51be52c2a4496dbfaf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/3c047888e83b1ba2c43c28085b12f11ac63fdb1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/3d16df3924b96fceef761178baf37996aacb86af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/455d152a37507d4724adff9b70dd6e8af7ac62ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/4573f892cee9fa7fc8b9a511cc0f5cd88c5716e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/467c85382ac99746fee4d5cf2961cc51a7dcb653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/487f1586f0758e0379e5b07954530768a545bc87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/48de2e34c0b3017bdc6ca035a43b94b5d66f723d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/493a69e71ecad6bb5466b774e87b5dff237f2acd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/4b1785045019cae71427b47fea35663f62c5a9d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/4ec5bc08b23ea26b653912b4db3c30605db555db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/55937bf3a954a57f8447a3e57016b118c9c7c3d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/5790fa206d4fcd6537052d3a7d16d75f08d9c57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/5b34fc56ad123f857d3ebbcfd83afa9a988520a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/5c35fbb857f814f67898b2401ce9d3cc8e2b55e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/5d4a103c50c8e9e0607af4feb388717fdb3ad5b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/5f01fc6a073afd242df720d6955f7b7cdca27eb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/603eb66f62e7359bfccb75889bd7c079338d1951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/647f72e019cf32c6b8b131d8a09425a6b1b50d17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/697513816ee5edc5f31739b9cde9eef20dfb8267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/703589db7f7d19df904cfc4979fed718f88c2859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/726f7904282b36e1f7e5551efd369503ec74479f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/7854aeaa57a06476d9465185b712dac63aad91eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/7bd9e5e61bf8cd02a56e2d7d85da9123e48031f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/7cbf328cea0ff2a0e03cac7b31142b9e832073b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/832a403ef99b8773a9e96b23eebb4fb42a0a5e1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/851e1cffa6d9adfac02f075877eaed91fa47ae94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/85d4ad13099727aa387962ebdcdf022d7d65bb15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/85fe8599bc199a13679db65a7c8972fc4d9c2f35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/871f6c1453912fad7ca785aec460e8fedb060e7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/8905669ab6a03550d51b27dd2c7810bc2958db35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/894c6096ba694fcd379493a7766f0c17ab52a317.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/8b494652b453e58589579250494257db4893433d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/8d6d3453b107ae25cc741f07310eb55790dd67cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/8f6ccc101839d9711cdaf4f4f1d836b1f8acdb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/94b66d1b6a4d7ba2bbb2bb87834e2e7fcc33a797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/973406fb318b3c4000611d4159b68515ad92ac34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/9c920b4743b9f8acad8aa9c76ece63171b4f91d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/9dd40da2c2cf5000453d544889dea99451435e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/a2d053a877f3a5861f04fb1a894c58f359ee7efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/a6850fe24fc91d05f9b210117b550ec6dec84410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/a837fca2bd060da0a6e72ca2fa670c726cb5f748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/a8383892b85a2c4c6184bb805e3bdc2a33a9fad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/a8f8d4a6127e0071b44df44ea2f2b8a62d9f244d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/aa6b694852555903e98ee36fabfcbb7d8b5b634e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/b18a06e7c40a4ad52cba19afb09644e94bf8e8e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/b87046746c8946ae645d236f9a7801e987a88ec8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/bba82e37a264909317b36d20b38dbdf9816d8e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/c18dc0a5aca3e015b3be872d35fd9af7aa6652b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/c38640cf17a394b106fa06cc5805d509b1dd6638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/c737c8b9f204955622b4d64f67896132be144005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/c7f8f4b22b9f5ba5f34ab3971a57318881bba7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/cc7a8fe4492cf6af42f7a1b1c843f667d319f8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/cc9ef93429fc6fb49bab378e24c0d22e8d679179.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d0d9517d0af83f421ef01d083cd55ab921c10c81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d119f8e6d615252578526f5decd38486284b4c88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d56aea0fb7c0fccb9d2c9e5b35e8cfc0bc95896b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d5ca7484912fe2290d412145a2cc1767774f6651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d7c27bbd302252e5e2ec46c310a4f1e547c46289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/d969b8930d8b6dccc2cbaa69a150782ccea64114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/da5c975be9013e52016c07e51cac4df6d68e3521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/dea66175bcd6ccdc45647ee07a00454471a5159e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/deafcb907a5abf3fb3aaf90420f11010c30b5a85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/e14789ecc43990fd4cded34a891093afb3ce1ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/e5ea0410ced16af884bea247b3f054660dce50ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/e74715564fbf90cb2ac25a1283d88a4c56ba5fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/e81579ddb8bce8ab7442e5d6b1d00aedffec80e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/eb1ff2e11535b31a6e068ec408b68812256c9aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/ece515231d660251d4b84bb595efde26f946f846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f0d3aefcc6c17d726cb07b6807a85e9573efc70b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f2af9649d32b5c06562a6fe6f3f93e105a494df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f495400fe28c77696c1ac77d027e9ec28363ce54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f6cb4fc9855e3e9a1ab8abe82dcbb054b0173615.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f7e9dbc7a3a1a761b83e5f0717ee26e5587d277f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/f9dc4eca32463c2a3cec97cdd43a4fa0c4d17455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/fc79d806bfc9f48b3da5bf500a54a37a42ecedc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/fd8de760a0da5d2d05368891f77a1503be198024.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-22 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2017-22/mix/input" + output: "/tmp/cccc-CC-MAIN-2017-22/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2017-26.yaml b/configs/cccc/mixer/CC-MAIN-2017-26.yaml new file mode 100644 index 00000000..5842ae54 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2017-26.yaml @@ -0,0 +1,833 @@ + +streams: + - name: cccc-CC-MAIN-2017-26 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/0535dafecc37b0b4820efed50dd6e32c67c4c64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/0c0636a46fb29a37d22135a9a1d86a0e4dc1d8a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/0d87cb7295ea06d582fa6ca90c1e718c0ca03216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/11d2c7e18cdb761f04ee7d257a5215a91d249bad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1358d7273c4d7b2dd1311ed8db5edd32f480b504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1467f82a39374bf00cbdc535553222df2cefdbde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/0ac279ced849afc090b0de991eeb44b223b9ccad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/21bf102ade683d32a6bb29a56253a76918dfbf8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/4d49359ce63cbf470753781a31157b92aacded8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/673e1c8ea4997c143df3f82c467187ac31203d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/c64611d00454b0a4b2a980db42c05d7aec1736e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319265.41/warc/efb00c5819775eabf6ea9922b3a63e5281bfd91f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319575.19/warc/31901595ccfbcd06d85f4a596289340ad55ba7cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319575.19/warc/364db914c86dc3e5eddd9e27272ad538c8bbeebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319575.19/warc/59c15f0bca8414ddd66d4530872ba5d646894ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319575.19/warc/f06db7ac16024249582f09e2c65f50a561ed6adf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319575.19/warc/f5e450ceb7acb5ef635854b2148a56e3790c0959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/4f39606ca01a8543eab90b6c697a18fd8e2b2b3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/4f5eff497f7cc3b64d1599fc4458311ffb4548aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/5c36ad6e431c89b49f7e6e90f519d4a4733adab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/6755959ef5f781fe98a102d0c49a611b1717d568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/800c436ace44370c02b03468e6b88ef99ec1b74d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/8f723653f755716e3d0ce20486173d5cf3c5765d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319636.73/warc/cb7f1457a0266fe8c697e4d2fa0af3c363ec5e6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/1e06c141177d4ddbfc09dcbf30dcffeadc98854f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/2267f22d99fd74daebf07f09a426c0040768ab4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/3c0cef0768e672d1fc0cb317d1cf83f1627776d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/4efe06438a4293972d3902b606a55c73512704f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/50c500482213deaa8e655bfe902f291707537535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/53e7cc2de40b1179df5161156e53fa93ace25e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319688.9/warc/5542282d68754821f6abb6faaa51899343ce884e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/17ed111567093740f6889422d170566ffcab8ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/2294867d0df53f4ae52a6c4b1a81c806dfd20002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/48bb3d6058babe361f7a7a7bc0668bd703c73082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/6ee80684b5ce789ffc478a5b11bc38c8dc9f42ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/b70b320347a6d7a017aa083d845bc186a001f938.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319902.52/warc/fb719719458235968d584d224e92c1d3c348500a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/21745c30a8de9e83c5dde43696f0db74cab9abb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/2499e3b323beb3d8bfb46f054b869078b45021ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/691d1910c43c167500cf9e26b72a572ea694ad4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/7ab0259242767da44da9084c99c7bf5dc5084bf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/b04ea60ec6a5daecee766b594fdee501dccc667b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319912.4/warc/dedfde48e3f4b6ca6823fff9452da0b2972ca5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/0caea6654602a43364c379c879ad106f22ad5c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/5fdecd8d023517140878ba8f483daab2e96e665c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/687f580ddf56f67a30706930f0b80504534d3194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/ba0b9fcedc3b3e00acab973905722fff427ac31e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/eb4b17e10e1cfffd5ace8f2e1c36ef8722471e7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319933.33/warc/f6ac7680231a2bbefc6574cead6b2d9005042d93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/083f3e30567ee0ec7fd6378a224af5e669c7aa9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/2632703d65ca924981a1bbcbf964c80f46ce82f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/566272a5e38ac377690c6f45cd1f22904e8d7585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/653d7d013d441267b2ac4d2e50c7f85346839f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/657d9b7f93ae94bfe30389f43e1caf2a33ead617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/773c79dec6f4d6446c6447ffa2c09582a0fa3918.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319943.55/warc/ca26d0edc77d58f4937628495fb6dbb417998733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/00dda92a0f6bb22b8c146aa80e168fed9ea431c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/0386e61cca4820aa09eb157f929d1a25e1a9a70a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/1e98a8ef1c3cc4c2d50c5a4ce38c08ce44cc1781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/7a004f5681857571afd231c0fac02cc56cc818b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/9910f2956cb6b5ed1214e74fa3db52e39f018274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/dbca9827f4e2ae75ca996a7516caa8f04be7bac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128319992.22/warc/ea7bc1cf2bf0886311394ab5d13e1ccc3261b711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/082350aae3aff79ef6e1d9a6a5c56d15a3337b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/1144e9ca6ac89c9ce690930f8b440ec2c5f5133c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/1e842525a72c1e99ea3f664f6a9314080cbba174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/4459ab52ffb6ea1da14d5c65388eb3e71be3a91c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/8a5216183c7f8aa35d2b75224daf1490fb26c719.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320003.94/warc/917d955f4f0cbaf90de6492a0fc364f99b89ef3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/3d11a13f85e1d3eedb44817a01fe1dec586cb3c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/51164d697b281ef1278a5ca1597ab58e47e77985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/74e92935225e808920a9a368d62d80de37fe69c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/826264ba50aa2b063a1d2f2b59d29d0530de6ca4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/b6aceb3a634b700552e804759aa0ed8dec42fbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320023.23/warc/e65c6b96bb21395684d51e9b04fb8689b509df07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/1cf44b6337626475111ee46af5bd98afb5d4dbbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/2883228f48e8bd524699c62df49d039c81acc18e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/55c477f816c602283b3a3c14b8026c247721e62a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/591cab9c73de7b4eab3d89b7c762ccc60a4652be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/695ca5bd86b8e2a5598c456bc814a518a0a3aebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320040.36/warc/c1def38b258c34188d2e89e6c6a121f4981c4aaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/14f4b276d8a0d312cfaa82e9a807c582125fac40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/22044625f48b277c034338e617da7cbfc80a6569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/645c7fd4ea7dffd99bd106d2071361d97b9c8320.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/d16b9af1d39e8addddf5f9eed2fa406e7e83b18d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/e8bc024411dd2a5bb20aa66d587a236e490794b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/ecdd1270b7f123a917550adad71151914e174a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320049.84/warc/fb84218638df71db55c80aa9c0041b4c135a6586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/5dd722f012a0563794eaa7144a085eed8f4fc9d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/9bea1610e6db0feeaa0ef712702b9ada1f1acfb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/a2ab64c6f6a37cc3450713475f981dc999856b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/a4720d9bdd2e67f7f1e0ad4979f9f540b01f41e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/b3acc363db67c03c457bd4818ffc28d30da4cd61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/cd5914ed9d02c9381bb36d6ddc76d1086216dde3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320057.96/warc/d33035576ed99b19353ec964faa297ef468ce820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/277e481fc2c03ed0f69d72012350794b2a3117c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/3b026e5615d4d1c277e76972d917bff257b67cad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/44736b5558e95caf02fb6fcbdd702bf840e63d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/781b2b180ec497bcfcfcd972ec7c832dcad0fc9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/87cef39fc116b14304b2a9fb23cc77e71ddb7316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320063.74/warc/ef3333b42b8540b163eabfc80f6652b66a1caa6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/0c4e6248ab46a455500fadf1f1aa097bf726b84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/0d478e9e5180d760a6b3bc337039becad86ca7e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/3b99d9d37517f67afbf0d6fc8a2b8427019d81f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/5818b7f90257c13ec42c37ec9667f672e55cd3e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/6dac6a93a45223ec0fd664050e951b03b1ad251d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320070.48/warc/d824f5346043253e17d48cf161db18b66c093b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/1df2265bc9aa8c9be7baea95ecb3d944cfe9265f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/26f05ceecb93e77bf591a5c286274e65fd44ff66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/814f15ff780a47f199080a0c22154180405421f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/a2c7f7235bb43999db6e484853064626fe1bdc86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/a694f1733ccdd0871c1d394ebce9b7e4df4cd4ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320077.32/warc/f97d17cdf297c07810260ca3256faac47dc12911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/01b1434d77fb6925231250ac2b1136d18e09a608.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/36489c86af84d6095c09d24a3da198bd34d48eb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/6c7ec817b71902d0eee2aec6467191250ec9d181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/6d842e82ce0e8465cd1b485c0646eac103062f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/94890c0fb6d4fe06a8609614657e06e6b41b568c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320130.7/warc/c73829658a8078ad57b74c5381743f832f19e3e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/3b8f37c961deb275920d3038d24ea387bf6312a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/79f814b0e849b73de6ae013a72113a4c2e7c398b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/a0c540fbb7dfa241dd8185416bdf4ac4daefedd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/ab18c9f2ba808ed3ba62c4277bbffc79f62e3a67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/af236467adb5da05229580164e1293140d9d469c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/c738ec6336f7ae59726b5ebc9c95b93dfe65193b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320174.58/warc/cc0eeb832a5d9ea75bca823e4e5bacd8b625ef82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/1b58bb41d9b85e59fd2e1fdf118ab9379df181c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/32bbdb777bc73c039e7d713d29bd950c80a29fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/4e567a3be8819550881c6d5421b69e035c59fcd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/8688d98a5b026acb439d8629041cb2a3b490e295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/bd320f278495beebd93d820a39d3d0f26d8f614e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320201.43/warc/e79ccdd306891d3facc81e4ff6bba3f2e7ec03a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/1ed94b1c42fea94400b8d1ab604acf5cb88813ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/9bc4e325096c64a6a6b79d32f0396029ffd37449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/a79e02f5e7eb985625f16272370aae37ca9826ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/c9eb4be502c3dafe0e1539894d16d50a524da236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/e5b7be304932d14ed7e50a5f9c3fc46db15ba75f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320206.44/warc/ea43646d76df188e7a04cddb7f23484cc1230de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/5a8fd2741cc230f15d5f1973c01bd3971ed8cf4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/6756a356df0b2eca14a4349041a19a2b2b7ed418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/784ba7bcacf4f6d5caa1fc0e616b1803b7650d2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/96b153b087d76a5a955014975db6937514294a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/979e3978e6e07002ccc028f7d8d7c007159f20e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320209.66/warc/d0d8bbf212cfae8c0d4dc2b1e404a81e01b85873.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/29d9371ab47fe5a1b0d827504b59f627607d155d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/77d02b55f85289a4d5cd40750fe59b6fa6aca63f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/84eec89ddd9a14f45be5f699fc23db9e469fa945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/a9c4eb88342b6333546df4667bba985c32a454f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/b8f523b9377d89b1f13c6c472d1905fad2896e64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320215.92/warc/cda5080ac12d22bd3a25d9683893ea8e7b6dc1ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/204a3bb8ad04c67c1b32785148d899bc0e7c862f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/bae5b1e26c868c3ff6f92d0efeb5e4289f80a221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/c3586be82c877695e8e202f8fc805df277827e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/c9bb94c00c0d64dde6c7fcead99505c465b39a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/d25f7b3b217a576e9b9067a0b237d9e278017955.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/e0760e9520b4e2883ed7b87766fba60102228e74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320226.61/warc/e68c49ca4bcb5902e02f08db6ba5b0f8d46d8a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/50f65e7d165935e157176e85290651f8c27c9502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/5cdba191f8098df280754c4a7a5b5698e9b581d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/7ab58ba8dd90696e78a9daf298f90f7a63942f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/84104c2f93494c19c55c392605d0b41fd3fbba6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/ed85a550efff7c6ddec1d03f1177571dbc28c5ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320227.27/warc/fcb86b63a76fd0b857c1bea0ccad0a2f9ac39fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/63fbcb9359251d9efa61ea2eac7ca9f8fbce9ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/785b4f7196f0fa1a6f6ab2b3e8932c788b29859d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/a19b796e580bedbdff4898debbb479ccfbccb040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/a8594ac1a579fc647f11fda7b96f352440017c1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/b767a94cff9954f4794720a6cfac32c2511f6e2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320243.11/warc/bbbdfbcfcfbb757bb497568d09f69708696ec47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/4fbb71bed84a8707f62bfd1b3af398f6d39eeb5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/5151e4b6893f6b36ce3ff9e0b3c1731ad5e7ca87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/5b20a5e6426f7f02f5b65bb9bc5aabaf75e612dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/90140eb1429adcd242709c4637a80d2c27c6cf35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/9ccd1c25ee88cd60e0f8002281a225a80c4b6e42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320257.16/warc/9e28a43908438e0ff3d97082929c83da85e2c823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/08eefc47e30cfe3272614b64cfa3fca0c7c92817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/0f6c9481de8dc0003b7a9b37414c4fa08a45662c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/740a281b69ffb3cf152ed6625541724e4be103de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/82e2420d9ad72398bfcf95c1ba04c4f65b562ebd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/b401a73e693e60574383efe64707fd0193dfdb41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320261.6/warc/e4bd4cc0bc5380af597ce372ba4b3d20f49d1451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/11af81b27c82311f7b2e697de4365edfa1479cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/216dcc595694c3becc984e61be4a9253b7a43cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/2b73991c3db0e8b4b3791b91406371044e5efb23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/38048b09d9bbf6622ea74573ef05f0c698ebe4b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/743bb5f62245f758478ee906e9a5a5baed30e50b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/8975c630366929f2e71dc50e3afb2dac9f0c1032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320263.78/warc/99507c5290d2ccab5e66dce5613aa4ff6ed4bf56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/1e638150e5e5ec8182c3ae26aeb496081556b450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/1f83346da1f0c65121e17a2367147b483466b517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/484c0fa4e95b4d0a8a297f414093481dc5b13e15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/69a0334f5d547c2985b83d77d3b7be51b33a416d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/f962432379164ab3073ef8f7ba754e730d4d1856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320264.42/warc/fc0b78f534d01524231c9b014a74ae05e8b048ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/589de79c0abdd580a9ef1090416fb4a1c8916510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/5c606bdc1bd4391a31823951a8c684c645299c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/8093fc522cb599f7849fbaf64533e3fc9e539a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/88954a116dfa6f23aa9429cb992f1ecb3f7007e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/91bee9bb131be1a7ef6d33a901dc0ffcd977c64d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320270.12/warc/cb32f3512faff8ae89790fae00703c71f25f7f1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/0fc9a93323d03171090608bc873d0a5efc547027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/574097a63ef8983f482669bf79cd3776356b3d81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/844b05ed6cd7ba43101d2779a86a71127738382d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/a58cc57e68f61b68bf174ad2dae964b081825f4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/ccb251e3e0fad63a311766c5128a77ac6f662df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320323.17/warc/cea05a6cfc6268581105590377ab561afac2782f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/47981d353ba4eeb244e396600e115d5588d68d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/4ac1eb39b2cb032ebf2d271b2e8251e824a93436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/4b092832386fe6ed0b61c6a8cb03ccc9d56c6fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/527b944594cb639f6cf7a75879a007e8def3744c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/a6f6d6909a04f5e856865d6484cccbc9a51cf604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320338.89/warc/e9d98e54f33d32ce63dea896018f957a1b14fee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/119a063f50aeef52509d8fba6701f314e59ffa76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/157432f41f958d705a226311aaf8a78e31c0f580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/23348f276ed507add382ca6f609107ca6ebe8d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/4e5e64b2573088da33ac3c5e634831b71d9fa589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/5b1bb7b1061e9a656a06e41c106ee88499f80697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/877062f01263bfb8b2b24958c03879cd84e74b6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320362.97/warc/c981d7bc17aa252a076dbfb47fc95968f73d475f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/24894c047435d1c01c14529b0726b8a9285ea3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/475a8b6b54e5245e2eb095ebafba2f5a5a048df2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/5bba8668ea97de765670203038df4583f2e848a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/a7197fe7ee78b7f51d1cb826d9bcc7b8966768bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/dbcda616f3fbaebccef4fa51365bb9db7dc6bad3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320368.57/warc/fbd4d6de4612dc6395ed2ccb610845c60a9ed49d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/969c38c7c8d1de5cad6bf2bad5321108a1e56870.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/97a29385d13d72bad7e58992addeab4f97bad114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/b63882e1275868260e18cd7ce74f56c7436294e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/c37d92491d79e9c9af47391e23e0e3dea3f98458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/dcc7afa8c022c16f9bbfe85043eeb5f2ff0b0145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320386.71/warc/f5d94c17d5b49668eaa00293f53e36e9f8dcf96a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/1388493cc9c6654e6bcb3de819acf7f4e2751fce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/181f8e0387bff5304629adb7551abc32dd1fee9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/3314bb0418765a7af226e80423189b0696a4c147.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/6c84473ab841d2704edf63fef60ef27bf4fcccd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/75848e398185bec7bedabd988fadb5da1ce56b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320395.62/warc/9ffe1ac4d536532ee0d90549ab6e296c72977b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/22a52de1f0351aaa67f596cf95985b297a9976a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/2943385454e5f173154971a2cd18be4e7689c688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/81e3e88a39d2c41eb16b424c15a57d4f42a1d8c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/c565f0d50604fe2c64843c0714eff93387286516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/cd26939c77efc8ab937a167c5d3fc262016a9e8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320438.53/warc/ddc723c364b9a4617234297ce3f282199e6623ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/5696eff20d5286fc6c85a8e57dfb33c98a3982eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/576392bc3b9879bff5986275458d31dfa28bdd92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/5fa5a34709ebc9dcd59dc579d711f15b480689cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/88f37ea10a4a2d0ded382d8d639eaeaa2b4e54e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/8f7990aa4ec0af37c4b287370245847ae12f0cff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/be1c1dcfbce1aafd414ba7422dddcd65c35ce295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320443.31/warc/c967c54ea3e012ad42e563c5c4664a5c8dcebaf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/2f5e41c222cad2dbad174702207cf23f9d029068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/3b6e9e86ffe58b415d9e6ad9e92834906c09abb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/3f667cabac34fae25e3685fea5a3dedb516c61ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/a338e7b2bd0f578688eacd49dfc46bf84649b2a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/ca4f6dae35239696b7d3731e207164ae212a59d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320476.39/warc/d5038abccc7179c735f1631ccfbd1bb965d03be2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/3246e53041ff46e287b90ad30e68e1143d32f453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/4ca3167ac2a2a5534f8a1185a2e82094d409a46d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/a40f2f6ead66a5c3e859dcb33f3611cc0df291d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/e62a4f8f8bf7ef54c66553be3351a83b3586fda9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/f53fc250afe2656080d67f8ea4d1ea0d666080e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320489.26/warc/f9cc81b92cd3629b2827e1e0d90495f38343cea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/11c67a1354b74a56fceeac586de6edcacc5a79d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/1b5bf58dd73c5577ac196c9f6ae82982db26b186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/3248fd059f21e5851668c9cd4ff3faca8413af11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/60f7bd2e0d9802822542edcbc1edf172295f5c48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/7b15dd3d175e66b5075eeaa2422cf2385e756be4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320491.13/warc/9ad91b4530157873b5ee459933182f0c8bf34ac3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/182a816919e24b7745a76bc6a29b2d285a33e374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/2963fb9a9b9cada853015165cb59a36ea2033294.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/3dcce4e567254760be8628ae4c31d6bbb1c549d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/6818cebc8ddc308f8e4edc74f0daa049edeafe5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/6e04ae60c0340f6d016f74a4773526a9966be4f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320532.88/warc/b9920c89cc0f066b3588c7feb2c25c27f7d58a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/0d2fe43d5c1483ddc2cec29982da824c4dc0f459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/1afb73932ce06ab28ce2bd9c97849fee0e3f2f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/45b049977250a2265c82606a7ada0c7ec5981b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/50ce6bb70f453647c92a02b0c2debbb034b9f3e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/a656ca8b3a07924038eeb155ed78446cc32a5aca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/c252e20c08502ae12f153cecd92e9d308eab6cd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320539.37/warc/cde3cedb2e5127400088b3b778c28dce33b5db7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/15217ffbee3ef425a1db021c0669b0bf21640489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/184d8ed9a1c9c5053869d1c469f5ee25503a1744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/795ae999537a06b3b2851a6b5a0413de627ca970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/86b7be2547aa1f033f9eeb33ef863e5a4c019a58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/cfd069c28b4d44f09ccd0a14cdcd4be53d1ede49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320545.67/warc/e09fae44212ac7401c2cb55fce13c9f6c55bee4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/532f1b674b7b74b7328492e4b6f071abb513f482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/56366356edec0ef6e21eaf1a5e7729288f890659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/66d157f7c54d675a83eb48c198124255a774fb9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/6bf43bb338186c8f5b67030ccba26ce1ef9a5436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/7545bee40c7266de51f42fb2f39b3a0662981296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320570.72/warc/cd4a7511147ad794a51b506b0ffaaaf5288f29c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/07cb1025f41c1d4888eb85f92405aa1331143c44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/2b0b3ed3cffe5217c486902bbfbda028df838e89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/2d71798d2a56d7aa805ad1e86aa1455adc6f6c21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/89f93e145bd0daf68d788c6da674d682a667bbb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/b485b10a176ccb335ff115a088dcbd451bd49c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320582.2/warc/b7866c25113e11bd3d1df7551c1c675f0be3b490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/059b9f6232c7482173bb36fec9ffcee5cec5c085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/3f05c437abf266dbbbdc0bd24e5fcb9503a59c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/7d1c0a47171f5a287b90defc93cd8715f7265661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/8a4cacfddddbdacfe669e368f44cb073d1db3ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/b15c78f92670eb4635af379b87dc08c08dc109c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320593.91/warc/c0329faa3b5c5ef7a321d12448cb184ac90c4a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/00ccc38484a64d433b10da4cdd7bbef24d9b79a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/084a6ec1f3f8d346aa8d16389d959348e9f13e0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/2b7b2b75e01a4bedf33abd3b79ce1397049f535d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/2fc292f5d5828e0003d40ef375996ba24c8f7ec4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/6ed41abb419a9479d4ded59ae730e23480676e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/e3dcdc0661e70f4a51379c98e7185988e0f4fc21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320595.24/warc/ed9e7f7f93bacd4c0cbff49978e087ef0f46ff24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/2a50b99289a9c17c50cae2bbbe0c626e5a08740e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/702316d21d5b5fb419247b55e26c6a2216fbee4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/747e2c08a45020185ded543a0638d5b1052222a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/a07d2cc0ddb81e8e02546883394e360465b1d17d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/c18ac0a9816db1ed31ad8acd7cec6ffb16f7761e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320666.34/warc/e43656d275fae87bc6c1f905b26a650de1dc4dbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/16d0f1135ff2c009ba1346d4515eeba1a29ec21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/315374ba5276adfc092d56d72885cb8b57418291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/7bb0af9be93e95efff860c641ad22c1668606576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/8c9be7718f3f9f06660962ba63c17cf54233b14b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/93effab152f2fb5644c616a9e9e67344b837ce03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320669.83/warc/b8870c36d527246e4a858cae2bdeb09d8eae6d7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/1e9f7ace36a14f6e47e6223ff558d945415418a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/276daa6cbdae3e81e5df5a616cf400734c584350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/2badbef416cf562668d007143ee2128d42f44ba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/459a5d75629ab230c9170fea96056ea392405545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/a357b9699d8646ad1bd86d99f685df7d8661600e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320679.64/warc/c53bc12c65c9c5d2ddd09aadc689546d6c23fcca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/2061b9250c7d43dc203b9cd238f45497ce315c2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/46953124d6802124e624370e93e802fcccd8f182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/488e93797c1f5e4fe8f14d118871bcf4e2f625e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/7be180653f894f0641f8ebe6b0226de61e19cf78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/80cd87f1942ca551c28e275de2c45f2ab6fe25f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320685.63/warc/e0dacea3ec7e1a718de71be0c6e4d10a74598994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/0670a7dabf584a3184a8cabb3fef64c242be6665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/53771b33bbd5c4d2d430532ac8b85af254d1ec83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/623264eb19e45f911020ceb300e0eed30f08f619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/9d54a8844bba618778978c7085c43908945827d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/a747e26e8f4248f1c32a686c18488ac2f3ffc996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/e381001166760d863b279609c03e6a8843da4a1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320695.49/warc/fa3414b304c2d532d1508bc68d3c354621700efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/2e7b0bbf76c1ca721017e14b117cbc3e4730787b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/3b42e1f3c3ffc8a7d2e5eae308e0405407e48c07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/6e5b494caf723f384e2192d926bb6b8ca49e83f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/a94fd73ad4f7b6b32ef7873c3e5bd59da69f955c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/b3300f39fed804b625fbbdfc986da48cf60625e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320707.69/warc/c3ffc774151183eeee9050dd209e332ee0ec018a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/2a77c0ec80e858239389c2e6a8dcd0c1fb4aa85d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/5180f446415740ff1556b89548d4db6fcbdf0418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/6b34c0f6eecf34cea8fb475cd9b2a2340de97b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/72bedb75f8e1bd9e906f31921b67492a93d1c4ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/9d6384b1db88fcf91b66481ec12ce82940f9fd7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320736.82/warc/c2ea23f9d2df752d924c0579b503ef1964dec02c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/38d94413e8d077e57be6bc358757da645067c6c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/39599ce01f4d7712de51fdef12cb9e9d97985268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/71eda191e36330ac39a52e8340b8a874961476d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/7efe72d8b49b9f5387b359d477cf07029174da4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/a7e7a306e8b0f8a973dcb8bcd11236e412c2fec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320763.95/warc/ce78958b5f618c88476a98272200896683ae1907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/078e91e7ba421d9e519e2eb8fe245646a13ef297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/13e7fa3f6065fc2e0b2662585e1810d372d64b9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/5cab6bf3d1fe9fe73327cd878195c39e171aadc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/c88bb9e60261b98869264f4ad14d64e6994f1699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/fb12c2df25b912957391321e4ccb20d68cf006af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320823.40/warc/fbbf87ca1a8b2e14792f58e10f3a97ff9355fa6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/29984a8c991cefb8596a971cbec6bc054800852e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/58ac9795f25e56972d46b2d323564c37953c75b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/a691dcef9a2f404d6637d61e4dda4a5df7b4ba4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/b13b3ff1d898fb8ba31dd4c29ce2a7219338fcef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/e6fd52cc94cc90491c531e9f2b87bcdbdeadb545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/ea2f0d0e2d905639a6e00e105c84c011b8fa399c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320841.35/warc/f6bfebea8acf42c352bfb24fd184ca413be05085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/1d57853bdef2f1e95d78368211d90d49bb009023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/25401f0b884922ab6de893fb2998ed622ab0f9c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/407e2f4540b47d3e03427254c6d4b8ab80615ff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/7b5ec802d9400a48365526eb38453e0eaad75466.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/a5f4c54ba4f69bf8add84332732380fc525150aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320863.60/warc/f27c8c95710a9e67687d6a9e9997cafb0a63b743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/05a12de2eac32782d590e727246dd0d8e69a5ed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/9561560546874d2abfc86bf128c4a29d856e9231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/b1d650365c9ce9b0a77710c97d9470e81e608e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/d11c0c38f1686f1c58aaa22939e179f1a1a6cfa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/d38eaf6e0d4b173e7bdba170227e91d3851500c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320865.14/warc/de1c035f3806e06b3b40a0f0d5aed371e7e9a03c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/2563c212b07fa2a29c8495d3e0bee01afaef10e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/2bcebda14eccb4377a5c6fcf144027bb1fa3693b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/871c7a908d8766a4f3892e3e5e94112bb82aa481.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/ae8cb4e24a715391b0cab2f191bf1d2255466fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/cd18352c143649087c24cee10a75f8f043ebb735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320869.68/warc/d145103edead3397ba200413924c1cbb57e9f8fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/32c4a02a4762d58115a279b5bfc6df825475f58c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/52e2887215387ec052b37eaaff066e3a47d9dcfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/566c613a247e54a3d1c3236b421b5e77d18a2859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/57f556f317594f2ce84dac816624ca1ed7ff4337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/6c4e6ee1d29009d6158c8b99fd13661561654d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320873.54/warc/7fd04350fbb21c850d7cb440e887a3e5f0b4832c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/06e376843dc938c7e904a404fbc258c5b8b7e4ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/1d59712669ab0a84bd12f99b1e7752d53ccd7bec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/2b927511833f181beeebe0c962a91d1c5bcea619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/367bb512459f32a03c56f7166eee4bd58ebd35b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/6a3c31420e5869cfacf15da201d43a4914dae788.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/840b8ffbddf6244e0a0bad325d081025d97b3896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320887.15/warc/de813e6235fb765caed2585b66bd2b12d9e81e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/16d081a5c538e0e8efca35d0d8aa50b517a8585a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/1e8d2acf00f29b9604596e6ba08ca851ace10325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/3e016292f92ad017d838b75cabe15aca133291ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/5f550e919ee4fe73906896b56ef650a40ccdeeb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/b83121a885bb1ee5b43791dcf067bf1a44b61807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320915.38/warc/f6e8e8b78362522c20178911963ba6732dcbc1eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/48edc134c6cd0187c3ffaf190dddf5e40b0ba2d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/4aaa0db61c8e61ee26d432905cd6d405ce013e06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/b89284929477ce8c0dc901c8a7bfe8565055639b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/ce242f4f015d9107ec2ba2cbba7f361117c70c94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/d97652a0854a067f9503fdb1920a0027470c2bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128320995.93/warc/f708129aa2f6b96a280263d22eb25a23240a2956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/66b142a7b668172a70c470c722b646f3a8516620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/6ca0b83196a1e32448d1fb2404f5ff4a032766f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/9b75ea0af68229143fc0737bee24d8c4e928d2ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/cff2362cbad26fcce24c927cbb17c83bc74d3c6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/d4c2df36cfafead0849c9cadeabf83f6cff23845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321025.86/warc/e36f95594757ca0f1998b36ced2d257be777b73a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/0c1b21e686d4034d01262d0f8e1a71877bae2284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/3815976d3e12581e8d41cfdf1cad9ca25d5c96ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/3b87ea9fa3e2c265f49b084d2189cc25b5e151a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/3ed915d3714b04321f09db37513b6540adbcfb5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/cfb79cfa0ec2af14a13a386d9cbc4dab1267ca2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321306.65/warc/e025930ce6b2269698de75757b23cc1f6bce1cdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/03350ad1ccbfbc23f7766e55deaeeab30f78fae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/2548a79cff6baf74f9a9498b9bfedaa50a7eeaf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/6d836d0225b045c4bc23a1b311c20144d6b20191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/95162debb3b76a6775af6bb1d6494b7e0a5a34f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/a086879b562937f892d89e456ca1dba91455328b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/d9d9228381dd321c9ad9a0b7e42d03bc56ebd7fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321309.25/warc/ec2a5f5d71915ac0fec1af906c64520fbeb89ae6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/3d0d4b684b790d92d7ef7ab51d756d4da74a26e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/71666491c6e3e9e03fc95eeb74747c81e4732e94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/b3fe4742f86c172239aabac9493ef2a5ba2dfcdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/b684916f0b15128766f0526a6efa959673be7f87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/bd51368b9913341111bf21bb272851477fbd77d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321410.90/warc/f4dae174118ef07c0ee2847f99f7e7a49cf26943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/52b95ac78ef551619027758c37c9bb0bdab7c230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/5effc9e45defc62ce4dd323057650af2e2af6b5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/6705644dad3783eb476ebfd2b90f03ed40560d0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/676c8201b23a1eb229f8c24c0037316836247be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/c4db8e3cd5f2147ee65a28405e6d0cfcee75dbfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321426.45/warc/d6253be3e853d0cfd7e4dff6755537469b761fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/075f53cd7f213aca7fbb4b69999cd1c1dc41efd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/2de61eb78acbce26e051c3c692817aa025afd02e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/9bfd14f514f4e490d37907a74f7a760851c4fe33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/9ec682e8753f4c6db0e5f9b589ca511f1580da26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/b3bb8398cd63a936506ea03a41bf07a7c8020d00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321458.47/warc/b3f56a08c0723152ca3ac769d52ed081c6d60ecd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/019025aca03532fc8678ebd6dbcd7af0d563b1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/401009b11e4f8963ba73bf3bbfa36399ab96d32e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/6b93df3175b32d39b16b91625ff42eb916170ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/be09c7f21b8ea2caab6cbf17e80e17c879439547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/c761895d3194e6cc44dd82bd7a0a9506c8c83256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321497.77/warc/d3f28d1f4ce384279dc09ab4788715840d0490b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/1176df86feed3d3f4721b656a01be37601890ffd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/4cd2eef72f88c636c3dfac4ab6ca55f8ca4f3403.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/5f6a60b937b388250ff4c52cb175a9ce5431b92c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/5fe653643f34115e6216a3361c6f67fbfe91cb3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/6502141c70b7cb20ceb2f9e59b937e51e36d11f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/6df7a9407fc84a40a073d0d5819f21cd418ac114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321536.20/warc/b98e34af6de25e81ed1f4f5a7d63d4d81d06bff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/4177a7e47036202d1ea2c9a492a0fbd56c55ec24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/5e78936ab3216f554e7b543996d1e73bfa02db1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/7fd8c65fd48c4b14e4590cbcb0267c892d7884dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/bd1ca85a47f1b11266d35a00d2bbfbbfa6914d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/cb4a3a3c7eca95623bcd69350ac401e444ba1a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321553.70/warc/fe4a32cb736ba2563489b5b7946eaeefdb3198d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/08ee0d8d211b598b97d1d72b6d91e1cc8c4eb0eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/57471881954868cb444bc53edb8166570b5e92bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/78436dc7f217c9e3fe5f4477e3c8d198f4b3ab61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/bcc6ac5e00dd843ab420a08b9756df7ad92599b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/caa1638cd9ddf98df9a976294c07777feb71eb7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321938.75/warc/d04b841f36829713419c9a9ac80b1b1b58347d68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/2e3b0773f03a6f3027d5d25ab03b2c559b891bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/811e5b139a5cf8f573e700a5db58e92d72c9a903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/a8f79a63056f3d0d813d2ee4bb6b96e4f822e962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/cb841263e979bf33d3ac8dd158d3e079f53b1e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/edc426208a3469895fcad6a3426d8873f56b8a3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128321961.50/warc/f6cf95d858f7549e8d2242dd5922507e33551c53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/01504fe4f3d8d5cf8a67e989b6534159bb25464b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/05c6a1b2751e78a0c8b7a8ce3724b37945ad33a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/4294f46837d420deeafe6bdf0e4554f86cc21b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/980f8cb8f2e19d4e1eb1137a73a56a31c80b00b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/c77992f5c71d2e408eefff5bab133a53a2b28ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322275.28/warc/d2b505e8c44ec09a48f3f2203d074e360cd811c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/41ac00e274b674ca6bde0f1599c49bf3e564898d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/43004891ec12c077c48604c6dca49aa132aa351f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/890062d9adf53ce38a67f24d2d457609af6191fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/a3a9da9560a912faad414be931ecbf1c16e6866a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/d9f962bb43c948ce775a5cba076962d300f46f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/dce966355e55c529805eabea7c6d7fb866672fb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322320.8/warc/ff3c416e9b43dc41c8990bf5185416dfd25e8f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/0f53d879a8df4aa922b048be45c086083102c371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/1a3ac1fb3ea7e98fb69fbfe16f2cc6510b715a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/1a6a3358634281a515876786c256c81fbca25cbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/59c788657fad7a4cd736e3dd0510e2ca8e7b7a1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/8e739f8de3efb54484d05b125f6d2992d3306409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322870.59/warc/b9313905af9b30bf1fdd0879d26bfc60fc26c0b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/7666351494903b05a86780920a5663252bf42948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/a939c1a4c90f1c8761fe2e4048fd9f288be4b0f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/b10866eb08cc7a3872ea6aa9b8f3c9a50348b790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/bba54c1b1c2652e7e2023f01af77987cd165b69e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/c81b0ccd08d219ed01f3000b2f2306faa7144e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128322873.10/warc/de8c3f610fd6ad3f723ca770227e0f41feb096bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/15929a193a42282f3c86f6a6460443b2211aad25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/41a6d0d71430354f9326518ab17c5c1ca05adcbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/4a3bde5709e4a4958db85ecbd4d337ba698a87a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/4c60144ff4a2ff3a7fa14babc27bfb2b3b289a7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/9f4602330047a8cb6ac6d72795154bce3ebfed22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323588.51/warc/b25b6ce669a9a53fe9486c03646c7a3fcc3aacfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/0b16599a9705a22efdab3825e372ca8efc1ef1ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/22534dbcef6ba17c6ccdafafe8944bf3cca21396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/6de7d979fcdd39074a5ea7072a3c3d6b9abfb80d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/7373bae97d6654b434bd2712dca4a8ba9a0b5111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/b7d6f9d479a150a154fd9dc54d2f00590e45e0f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323604.1/warc/f5d68b457509858f3b88f06a6eb2ba80c3b57de1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/3044b74216cb4ef101427e9055a6e83fb266b9e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/41c454fa306c2c5c92e3592fd840888938652a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/6828d5472ef066186a14fd81487c4f2112e81b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/87c44cc3b7248ffebf6482079095d6b86798e59d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/9ad9fea4201c4778e09d62d81db5d7fd931e2528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/a98259084d782ef763b0c694641f31295f4646c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323680.18/warc/ea5b2ae24546b3585acab9847c9683f87cedf04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/254373b469cad7d3047b724a1a03fe67cea6b0ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/b7f5583eb8f31a0295e5bdd3d1a451132c2023a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/bfa50bb238427d2ce09dc8ac69d07de0eee03268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/d56ac387fb6ceabe19ae506e3918b5f409f68c44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/e9c09bad6256f3e0aee8235209c7f848c79121a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323682.21/warc/fee7f799c3a627ade11758530933f533280437e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/325cd103d3877fb3bf8f0f0d0c0cb52f31811f5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/43bf971fb06d7dda2d67699c9de192b6b102c900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/4e3429f4606c6c269eb5938bd09192133ea50a74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/ccb6c54da61173152897c10aaa2efc522d429473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/e0d14e89bfcc37722ceb2cbfb8abf9874804d9fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323711.85/warc/f87a0287ecc1fb7b26e2ea41bfd1d4c4668d0d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/41e92fd4994d1084eb60007e1a626ff21ee6c69a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/60e3032ce1c2fd89534000021bd7d03102bf57be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/79cd121f9f13e6bf76110759c52d3f8de2a9d1bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/8e345fdbfefa27bf1022231e0296c2e619af9f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/91674d262086c067ccd1aaefcc54ca3fe40a5d28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323721.80/warc/e6219277175f46aaade25d39209ab5fc1e7a2edd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/39f41d5cfbfd5a1ef509d8e876c0200601725991.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/5a850c609dce43b5916f9e63373ca908327adc73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/a351c068e6e6db0bb7e1f4549a59021482b05458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/aaec1fc8857b7ba486c731cb682e851e8b9373d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/d52ad6732787cb575258d6424d3ef812433048ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323730.30/warc/d67838232db5255ae5b09eae5260f5a70749f2a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/0a4f2afc871c7e707678b97e3bb5c2eb4348039f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/2a13f495dee57191acd175b2c5602bd7f6b64b81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/5a937f318963c0a6b3eac041c4daf1ae81ee6396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/6f280adbd5c5f47a176abec6c6f0ed986d6efd91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/913826c4e44cbc1d1ebd0eb3eb91c2de61f33e42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/c9d4120ef8b061528b2f62e4eeb221ac3c6354e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323801.5/warc/fc432afa5443b9000ccbf79ea99fb2055aca36ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/331b0a83c28d409ba2f9c027796338c5d9e7725a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/518abc547e9ccdf15c91a80b5dd80207c1e2d591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/706d9e0c42319053f63a6d05bc6870a9c5307edf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/85e7ea792b3a9df8796b68172ee4e5901bb04ce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/cff6907b38b71eeb50b386ce2e6a4089845a4508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323807.79/warc/f73d8fa51f5a74dce87d781eba073b963ea8646b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/00b136b2b7dd7161dc40a9ab918d8a0c488b71b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/79828007b249031d0e36d625574ba8f8e419c1f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/90a5d7ff42c0eeaa171afe1460c9d21035b370a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/b12f4146f4237e3f44c55ee6038a05a4f5a37d1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/c277aff1ea5796806b2621a2c9bda886f7494be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323808.56/warc/cd9783eaa6294477d6c7cc429e4e65e44cbc65e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/235561c38990414bca31ccbfbb3e866e66de28ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/76c2f6912c6b01d52afcfa6abcc6774bae7f082c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/778ee9b9fe7f180ae107457c30a9a413f35c9b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/ad21ecc96974af3e76e5970788e488e7c92ac4ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/bdf082ad7c4e34fcdfc624a1d9d7d10d8615f05d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323842.29/warc/f2e6b5573e565565c3b07fa43e9ff1b03705d3c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/17a386a9813e65dc722b9bdcc0692cebf70ee588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/61c666a8ef4617c11116b33149a35ec6dfb381b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/8b230175584a11cba0c9402caf80d5f5ae913962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/9ee66246ee629a6971c7540490b7f878868f9683.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/d4a3b123f707201c7f130ff42f6b583c7e16388f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323864.76/warc/d877ad1195937aa423968365d1989fb5968ad2af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/049299d38c2ca8e55aa7ffc78a67ff57623200a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/1248cd0ee681e1cba84b1667e3e4046bb013beb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/2fd3b049cd7af88ca1eaabc8887fb7ee1306660c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/5af1067cc94f74173859e141490cc032d7cb1468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/603882dbdd07290cec73f2c99465f95f693a7dfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/683f63e2a02b6d615e9d115e5ab093ec27a8e4eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323870.46/warc/a23744217936a16198c377e32f8909c23d15fa9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/5de0b7076031e580540ba7766bdd7635a45a21a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/838e9654e60f950c368d2b21f150782a4808b41d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/8d5f5d457e74e3c72a7d3e704f1cd825d6462b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/a6ed88badfdebe4ad905fe788af88224f6201e74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/fc14a99e74ea876e2a5827fa5eb8e4135ef5682b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323889.3/warc/fcb4d49b6af5b6656a40f06328e3f1398ec84b0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/23100db67fb0c673dcf96b05a58cac436e6c0490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/42d7a224f3408c4d3d07cc281d187bb704799db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/5abcfd95d7d46962278adfc9de679bfd039d39fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/62fc9df0c3d085d8df3bb87764a8b0829298eedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/a075f4204ac30b6646dc43589aa9eca6915b84d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323895.99/warc/e3d8a018709f43c9e8efbbab179b4abb8a977567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/38ca9da3b14bf96528b12bbbd76e0662e94107a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/7a63b953ab677d71c1a4b0f2e95a3f586a24bb6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/88fe07c104443d9b8992e0b123295c73d651f3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/98c2c4b1a39c916e74bd8137226f0b82d7004e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/b097cc328cc96908679bfc3ded8ca7ba96e89404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323908.87/warc/c15963992cba0f99a5ab5f300fcc8500d5fe82e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/17504a832e8c81d2a9196d6f5b5803934164348f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/29c02cb8ed1b5e8d4b88304d9cbe20b4684e53fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/86d190d88df92cb5edb8c92277906263c11469dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/b02cf00dd598424eee1c3ad0db4c53d5d1903050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/b54ea3d4fc83afab5945fe4447ecc2dbbcacb5f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128323970.81/warc/e83eda467c9f16bb509fcff36929bcd7fd49ec42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/138fb4bc1cf47ff9f8e21aaf401aecbea1dd30dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/1fab41fc6a4aba77eab5c9eced06271d12517859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/4f616c0b0ddba6212fd08a2daaa614cb51a40acc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/56c461f908d7da7b0ba8780b8ec925a2f9f4c1a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/8e8bfe7a9dc35268e47f694a8de17f939fd3b608.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/98bfec8aa844829096d9162226856604d9b46fce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329344.98/warc/e1876156e00f4b030508f62945ba09a762f48fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/0aa7630e8caa014751321fb05005244ddfc7df32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/2ec122809cc1c8bb2abd2df523072c48bfc92c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/4eb1024c8355061b13911df3891614970533fedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/78b8c803d2db45c1750501c09ff836cae44f1eb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/cc86433d64ddac0c42cae515ffd7ecabb94494b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/dd36d570806a364645883c8c1f070e2ddfbcf93b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1498128329372.0/warc/e0fe4fe6905e051eeef36a56c196097a361d8a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/157dad41aba2934dbfa0eff685755021fc4c8641.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/176fe04c5091e0d9a2113f5f09a43236818b8434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/183f1c907fd85fc06f78257fb55089b4e0ca1068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/18ad4c8191390ca9f90a2fe3d2ac3adf86aebbff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1d1561222880a57610b40d4408db6ac8aeeab9a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/1ecde3f46a703c4f5c70d4dfcbc67340cf543a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/261e0482668fb2eb3c99a06a432b2c22881ba6a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/26a744849642af09c2b2fc163c470fcd61600543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/26ab09bec69315d9da3e38b1199d972cb2a228a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/280a680d43f667beb6043fe647df03e6673945cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/284bd638fbab9c85dbd5fab46a9821ee52f7f933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/28fedaacc97490e361e95b495e31f904684c59f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/2a46b3022c003c12e2fc7287838f9e5d1b1645a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/2cc75c1407ca1133559877cdb96f6ef3147c3aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/3172175ba301fc87949cce5c7e2f4660204b529e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/328d7672dd7393971441c4b53731406f6db35f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/336cb6b982bec9acc6a09fbc8450b66ac6f6b273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/3a33c9c17d77131a72d37cd9235e29c94f800fd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/3e090eff07b1aef2b0741fa84b129ce1b8d56962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/41b1d4b37ff9bfd997e3939a0a98fb8699c845a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/4260ffaf843e32f751531f06ea92cf3ead5f7ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/44b788b4027426b6940f16f831e55c5e83126055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/4aa786580981f07043a1262fcb14778a95a7640a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/4ae92d0b67da1095fbaed144d9b428e4905e67f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/4c42272bc660a9ed40aece8295333887add4ff7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/508e4088bd80321adbf15c5dc3746dee70da7e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/54f06e25cc2d62ae5eeab103b9f1ae6a191568ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/58e1ea0dc2e3ea68779a87ea25431ce93b97bb58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/592f94c705592bdc7301720ec759be576b9e70a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/5fcd4f56d7575f958ee4446daf357de479bfa41e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/61711a244d37f8c3ad78119ee289f03495fa236c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6772a4bf1938253bbd6ae81787ebafe473650e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6875c9025f63aa1f31abe452ae3f4e57913160a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6b6c68fac0f72a0168356e92a0ef2bb7cc21db1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6b9dda2b89e101be0a68546ab17f8f65f64258cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6c392a22875dee4ede6e6b7c0be21682cd1e3635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/6d3223485d19cf7451fe9c6ab2e29854e87198fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/730226f53cf49363b33951dc96eefc984cb80b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/73a376fab06acff5e373b23caa14c96f54a11aec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/74e9c4ea81741dab7aaf76777accaf0a26e1dfef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/77069c0c14e3ba358f3645c63c44cd3284312529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/7f3c1e849f9e6e62c4bdf2476c5ffda2c41b235a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/8024cf0dc413721f3d6e5177775ab5a1e66448cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/87884243c26e1dcc8328725de5e835b8ec3fe2f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/8c71a472a1f51ff4655bfa5ec867188853984c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/9370442b4080aeb64e8b20bda2fb35fb055e6d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/95b2a520fb6b3bf9b7485cba654e5c1fb6c3ec03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/96ce83f8e786aa4397496fe91e20254f91495d88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/97855d5f04e72b2a2b05a9330da5144d086c0214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/97fcbef3f81721cf42c66f2e3d7c6831f648caa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/999a4a2536e05e44dcc45e7bc68354780cacc89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/a0f0e9becc5abe51f8ac5f349f9975e75c1d81ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/a3d9304c8ef81304a020a2998122a292732ab31f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/a57adef05e4b8639af8fc1ceed376d9ec88ea8b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/a6ffbadde861ec55765bfc3ab83ce87ad3599131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/a9f4886cd5df58653f15deb4cd62789e0e6b2bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/ada4e17b883ff5df3bb72adc42e3fc618aaa785e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/adb98a785d33c61bbae07e250e45b665dffa24d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/ade63338b25df326ff42316d2f82cb1a5e476345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b39a7f536cad413986b51b05ffde33db65714b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b4874b1a50726d9273064919376edaefde5662ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b4e803c667d6d4d7a281d6203fa6a995e4140433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b709f1ffef0a289615f5cb1e605cfe962febc416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b71c6e4cb26d15c05e859d16e6b12a159591dce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/b913b3abaaccf5eefd2f8dd67609f7b78065663e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c0f5e9b1b108f5c33048f3060907e803e5a1273e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c337cb0085040c007df244a2b35fd2d4aabcdf79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c49525dbb04ae93349d5a34cc7576f7258bb0844.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c6210b5ed6a84f4a56abd7779bd5893b2d74ce22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c6ee80110bd2299f8bd32f9aa2c89447904c8a7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c8639f8458b1e41b7f17c29f44d95d53c4141baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/c8846ee9030baf26e5374348fd833777cc290e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/cb660b9da9e1b6bad61dce26e904d612b360b15f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/ccdd2a5af5332b9c1e5498fe722bd3e647bfa081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/d04c1ce12f5547070b3e65dc6a715c4274eaabdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/d0e41eda6f061571643447bb736303d430a05dd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/d4152440482ee760f7a97fd314b03524aed5eedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/daca809123a77e3cc423f2b3a2f161370fb2d12f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e18bedd7a971a2a75bf02ecacb1048f58e4f8f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e210826fe4f02fea0edc1bd2e2809391da9eaa1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e5d73eab40f77bc5ba8a628fa18ef2132eab0d66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e6168478138de8ddcc3f53f296b05795ace4c0e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e7e6bc55be0ab4f4f053a71e491aec2a030aed1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/e91f55e35b0b1a034e0e4c866c6f1cd4b29c33e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/eb2e3923035bffcd4f7f55288f8dcf0f17a0893e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/eefa47b11ddd7ed6c9a85154f808190a38f22354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/efdcdc66eaefd9fc0a9c0d1e1fe63aaa78b3a335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/f3f9631a55b429bf0c9cfab15b9c7d0f068b6b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/f7b638fc2c4a1800ec30afa56d748b29410b2469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/fe3ee0f7ab2b69a648b3e7919097e4ce16f353ad.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-26 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2017-26/mix/input" + output: "/tmp/cccc-CC-MAIN-2017-26/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2017-51.yaml b/configs/cccc/mixer/CC-MAIN-2017-51.yaml new file mode 100644 index 00000000..84cab926 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2017-51.yaml @@ -0,0 +1,506 @@ + +streams: + - name: cccc-CC-MAIN-2017-51 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/19012816226afc7ec773daf753b43f052f4cf775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/51a652026ab6fd24950eabbafa36bf22226c8d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/56df0e2c472deddbd36dd33edac8fe4c64670044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/5912e4e84c3a03fa2416361cdd27363cae58313c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/7379e80f09fc76081eddf84a79e1f5bb20893dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/a1ae26a39c74d120f187b40daf9be6748ea045c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/e56436663ce5d62799b62ef5812b164f5ce9fa70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948511435.4/warc/f9386f2455e3a2ad57354d39a6603aea89c9edfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/072aa9d9f05cc2385438d7c74ab7499179f22765.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/3ef44801ea07d66aedcc5d3d0ed64e96252fe6ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/449b5386dee48908f95ca5d995cc1ce86d653ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/4abb693760b56150e06e78788ebdd02f33f95bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/508a8a3a5e0eab77d905ab27e4345b4302a80228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/ad5dfbcb8124cae3229bff5b2fbb4c34ea2e3b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/bdafca70add3fdf995fae8cf98a1a4815217b05f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512054.0/warc/e778ff2fbbf8d38bbbd249937220603f1b930bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/244dacb5bd596f5cface576658e7bf3e7f2710c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/4bdd3ea7db6fc2c97398eb71e079541d466b3bcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/632eb3cd621e3c53ba6df97dc657d2e42767ab71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/687e373830ed946e325942810722375c6824ebff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/9f8d2209f493857a189354c98275a09f84c696e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/a2b53a1db2448ffdd1c879a08f801b1a4cad406b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/a5412a652fca3ac0176d588730a244fee8ce5b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512121.15/warc/cf02be2be367459b5d021edae32c3482a8422b21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/0690f86f0fe42112a841ecc7b9bd47c7ba05af56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/36878614a78f406dc4eff29c995dfb67d4826eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/3aa824eb1a1244783f824d41f1300a5c27a861ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/4a5fe5d3e10bde52744c9291a9630a27f7861f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/98174f2dc10cc62054cab7caebf036e3b621cf19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/a0201ffa46a1069972cd1064790aa53b22d65fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/cfd09066812d40645f5953e56bfda822cecf8d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512208.1/warc/e77820d9dba6e00773b75cfacb62c1477c2427b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/033c3b4f5d555220b0581465fbca0c074996f804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/0ad23aa7a176b6a9ebba856428577fce5563d1c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/45e04221104071c3d139ac4b455eeeb43afcf525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/464b48a5b49161c80a75e1610ef50eaa7049c4e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/5ea10c0e24a5088397d0721c6f829f11f3579edb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/9c235db8fbf6dcfd01afa54d93e0dde534662a21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/d5fcc297cdd7179d1d4d1dc33303417712827306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948512584.10/warc/e1eeeaf97599f57695ca0cf60ca2e903bc98c41a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/0c2f0ba175fa405b9c53930917472cd5310e6485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/5a4b0cc27843ca87df0cb479943731a0ee63fa98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/75a3f5d7ab96e852906b88d20f651dde26dfaf44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/78e19221333ba161489d1d2c3a674790b861e064.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/79c8de4a69072d51877a9d3d69e74f0d0bab5c79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/a4ec4d54976b8ef0235eb9b9145c85e939ebf6ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/a8be809845920b6c893cf0e8f16bcacea9b8c075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513330.14/warc/c1b6438118435a37fc80062ca02c38af8cfaa021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/14cb010105dc814c4f444a426d80ba18e2fe85c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/74be57d82131ba1f59d89f3fc5e2f6ce5ccf69d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/9580bea6a18c11473563fe1d69a410ac2501e564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/a6b7e1735c8d5eb0ddde961fd11c3a5e972d4f9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/c0308353804e885b5a9fc576a0c75aa8e838ee79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/c12ba1fb3d13c2e0c24e2e1a90f8a021f9d4c93e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/eb7fd78b1245b7a64f89f2d77430b68f7b3730ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513478.11/warc/edc83e68f1125fe56782280a88b76f66856c31db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/46321ed8cbd8c995c95d42558cc398009fc8a3a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/6d5f9b1f895853e962e1ce51e19bf93e1fa2a0af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/84f4765709eeb13a2387a0c3a27ddbfacb25fe6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/a7403e07869e52c395a056d85ceede340c3ff232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/c69ca3a0849be8583fe627813a63f7d501a4f155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/d11dc5eaf60d4b72e1883803a6ef2b65f26ed5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/dd8176f536af6796d8d7b7d51e12852c43835768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513512.31/warc/f46032372a582eb9ad96d263ee02b0e93d580aae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/28fead0ab4871c6820555e051ff18be7e5aaada5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/3ae4b88b0137be600b29954d2699944cbb62b9f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/6723dbb44ff95d3dfb00ca68dcec7882dd66906c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/7c6fc41e5b9cc28cfaae08c337af6f489ab0aba3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/95e27475a1d71e1adb4f3ca8efa1d63240e0a743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/9d358281b22ce524fcf3a079c4c9e67356160e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/a85abeb5efc19299b76a4f5b7f1c58a0c3e6bc37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513611.22/warc/d9731b16dc407cc51e1494bf6c4a69a74d80af28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/02a0ad76b8cf6db702067df3a1b8f23cd9b81ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/4dce7506b92a7a108df948ffaab55655a6b588ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/5f55d801e866de18ad436750b5e15653095e5d44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/736559fe5695297d2c930af3a54ea758385f5237.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/7632bed0d5d7d5d2eeaf95f81922cda9a4325379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/80c973bf1b1f0b07deb5b150509fe2b4add0b05f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/9e3ae8179e8a09d960e06b3c68a972dfaa0645f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513784.2/warc/e07f459f588985d58213f2bafb4883b7d9a7c139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/12240540204511d35e0ad0c33d93c88fe46480fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/39d4ea6101e783769249ff14784e37125d40314b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/498cc390e93eb03ce8b068ceccb39889a2318245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/a4e91a0d1c2d8154dabcb613694f6941ebe08797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/a7b48ec810b1118ba62ccc2b376301343d1f96ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/b427ff2a98c2bb9b37b390e0c0b6b53ac5058c75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/bac71596c875685542daca025f6ab829ac17195f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948513866.9/warc/ecebcafb1f90bb999fab1b90a0053a283d6acaf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/4b91d0a49a167195021c0eb3f22b76dda7553f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/57d684bea5e7f7ec1e856622e373a90824f02d77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/727e19d14a508b0d446c160c9edcc72a78b25e18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/78d99a49620166b3f5bb0fc26e7097068c3f76d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/9b7e69e512e3b3c2596cd4d13c17acf27d061571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/ab3efba46efaefd4d68cf1e3e90c7cdee5ac6b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/c0843e14ae8f17599329384af27ab8e0de6a4fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514051.18/warc/c0a16d4ee5c8a780902910d9f7dcc561677de4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/49f26dfa4699356652e4ad095785355022ebf171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/601abb86b8966661ea174136dfa9b226bd857083.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/71506739b06c0e99b66ad8f00b5b595eb817dc16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/93a24cbc04d0fd44ef2c1067680d4ef6878eab84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/b306dc3dbbee5bf20841598d7115502d1e6fad4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/b8ab02f315b1ad4f6dd7b0ace511ed0cb7420a2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/c1fcf6f73a85cfaf89acf7f9cecaa332ff34d1f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514113.3/warc/f3fbfdd475d577f89fd33bf8db7eb1695f1df142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/2778f3d6e7242112151daa25eef4146765d3dcef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/5d022c129eb59af4cf0d5e57868bfe6edb19f461.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/620ffb80c85a517261bee42d235054d3d5529f7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/83877b4a7bda2a61d434b8ea7c8f426e10f9f62e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/977f665e6bb7ef22ff0ae4822d5b8fb3dc0585ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/e88428f080a8aab39689d4d2ac16ebb75498e80f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/ef9007b94548117135a02a917a1ba26c417ceceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514238.17/warc/f4c67e727f6d9c58f8ddb95ea22308e0cc2f53a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/4d2223fa9b0892789c8e2408ae6fa4ef0e2c76a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/6a1bd3717b2299627217bfd64bd60612d47639b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/6d3fe8b180ace36bc93e1297fc552047f7fdc3f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/9c87513dd862bd257f2a4929b614d6cca40e0c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/a3f0ddec98d03b849595d705a75e410e676967ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/ba30c27f08ddd87a9bf59599751c83e51327798d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/c8b7413b372cf648c8eee842ea27c6fe8db4b3af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948514250.21/warc/ed406ab343c9355f867bcfcc8a86776161d2fbb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/0c037679e5b17a72263c2c540ee4f6bcfbf3156f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/2fea4a3b8398011d36f162af1d39589bd3610faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/409e7c1390355ba61e63bf74dddb6aad61daa534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/a28d52f184d095b480f18ed06026b4f5f2ab96fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/b986613ea1043d407c8cbd55f9c5735384623fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/c62214612929ec035d1abf7179dd9158ba27107e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/c8568b405265874d547061eca3eefa237b4ced2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515165.6/warc/e2da57de34d3a276e1498a08223acafa645ae1da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/172178a6cbb279d2a33ed9a14fa8661453481259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/27f2044578f4618e3e6e45bf9cc9bfc14e432dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/3171c7447dcae19a6c11ade884f05d7a203b89b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/5a9c7889c52672bfdfb774526f5cd94a9569b201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/6a74e5e570319078133445cfddcf61213b2a264e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/7dd0205378699907df7d64efd84f00e1f1de95c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/88dc418c6f2fb103f3c2ad2d3e2ac5613fc4f3b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515309.5/warc/946cc33225ae8c156daed65acfc0d478d1cb0f93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/0603cd069d0eda52bf20d01612f597c90c7ce5e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/2c6c3a6824165d95307f76c023bd5377ddd81d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/53b3aff8e3e35564e9393f787c46d6d0dda53b38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/7edfb30f351c1200df7cd46e88c9fa1e9f825ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/9115a1238a37c5dfbda328813ce71ebeb6f29b72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/94030096738b14e868e94699275fc417d6913230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/9de2a6470d473a199dd4d81fbed67a01a0815c43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515311.25/warc/d77171e4341c147affb41855f6c141b80a2b5794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/03dbfe97b22c0e2ef32a58e2aae5ec3f1168d43c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/0e9b01487e40c59ccd124bdf9b6ce8a8a8426058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/29753cead6b14494ea88fe421305529f5a552732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/9c2c3db38c200fd85891e2e41b4cf0598afb18fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/9c6d5122df5d428a673789488671fc3c60d403bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/d8a876aa72490750c22b56bf5ba07ed544355a85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/da3a5ee4eca677e17b40c12d72d06e715a65b4ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948515313.13/warc/e52715880a11c2713ddade2accdb17d13775e6e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/031235c2bf36568f767ac8ebfe1b67ce8eb86d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/22fa09cf9381e9afa15126e0537efeb132f58c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/5f9722931e858dc0281d9672e656d23834b21d21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/7cde5b640071d4600ed72d665b91714078c9fdcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/96440dee0face7e306efc778404e1b278c6fbf0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/f0b3eeea323302250d72df79e5822a39775c7f33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/f467fa45cdf24e498f785e01fb4124310281a252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948516843.8/warc/fc58cb3d5aac6ede8bf1a8e6e84d220057c2faf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/2908489c9a27ec38388c7a92469fe7cae476c823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/3c04532dc05bce33a2dbfb84f61eeac7a4a87960.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/4165c3a1c606c1d99c498199ee45dfbf9f03b208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/7cde2275fa5a8a4642c89e9592c660034b699db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/93cc6e790dfa0bafdf21cf2103463a5d4c4900e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/b3d19ea4626373fdaff0a5914455133a7c795793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/ed0df46c9fd864d89640f73d58f51cdff2c9febe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517181.32/warc/efa4fcace9519fd366a322004cf9bcd532a83867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/1aea24d48ffd2ee4bdde5a686a189d8810af696c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/471bff1497b64ac6dd2eacff49d338179b22900c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/6905bf02b8f3474324f61c4bf62aaca53b15347d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/7cab9c43dc7e23e08af12d89f5eae634ad8418e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/9469fb43768c0c96e2a35282f4efecba7259950c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/ba28ce15a13bcec0cfd5a72af12b3cbdb28518de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/ce9759721533e4fa3832ac73c5aaf73447d86f52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517350.12/warc/dcd1cbd8587360759cfce7a8fce882b15d30aa0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/022e6cecafd0af1b77de3123d97f02feb9e30832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/5bc2206772ef2f20fd815d3822ac39406b26d368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/68a38d3ac2f3b68fe1923de280d06a06ddf00322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/8917effc1316c221ea02560aa81867897ab60040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/9346d59132ca0574e528d40119253fb68188f576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/9ef9f08587986788e501130f4c70051a77ac292e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/d150b1949f526aab54bcbc79169197cbc06bc2d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517845.16/warc/dcb7af36067d64510bb5de4c603db3bfc59b9e78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/21fbd3ab078c45a8fdb61d03bfacf884a1d37b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/48e02786ac3d662e35c998567974be47ce9596d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/6876780138107266e800e756d8880682b92d2fde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/7282928c954610d87c197647f662453ad52c51fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/9d103f2abfd2186f11d5ef8751c60e66ef820a87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/b93456faa8cd4f5958b258421bb3d389082eaf74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/ee93c6a6823c6a09d1cfc0ef681cdd84e4f23a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948517917.20/warc/f74e123240a52ed1f6a1382f263584f4d9913551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/1f47a0dae8953762c0d181dcb3d85a58bacda21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/2c0f99551c5cfc256f8e3b9e5f5e711a043928cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/9088ca9059191682b5f71b10232da4d865111d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/9359540cf8ea9f5a4808a631e04288c74fdd0270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/9e8a12b2b49d98dafe4ba4f6a75f85bce01187c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/a1bf9a3b593a1910d3e9b4dbb80baeec15f0da76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/de86b1c9e2702cbc366b9951fb92409c5b868767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948519776.34/warc/e0350ab672e04085d9bccb275b63b35396776da4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/2522aa54fca5cbb0215d39fb6d5e8f90fbb3c579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/378b75f2aba1cae32cf495ae69ef3d98071e6436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/41e7067c9b7d743ccf35242ab18bb4d23862f3e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/4ec20e5980a0d35dd0c3737d8b9218dd1b397a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/519871e8d40303eb99785e6046f124a0b0b13992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/57b7f466e868a3ea88e88c97f732b86a7502a2f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/ca1567185b849228113b6ab69593099b1e96d501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520042.35/warc/d25bc42054333fc9dd4b5f7422d463d6cbf054d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/01f86581894dd41f6ec489a8fe6a3c2a50c2bfe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/0f272ae953112fbc3ac4074f026d3e63affcbcf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/1fa36a4e82a9621f130773dc86f5d74e6a76e2cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/3337af6bb5a628d4a3a4a553362c7f36c6994221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/3a05634d84fc2e14477ffda6d3289328e3557050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/48bc74a00ae3a540e50f57dc469d16eec1e2fcb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/8f366d2da711f255cf5335b805ee940930a052e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948520218.49/warc/fc479bf6e493200ebf8e4b4d8596150cb832d03a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/43ab642ed3bdc0c131e7035dbaa6feded26fd7b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/47eeb3b9f26f69b74c22f6f28422a44c7efa0224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/627b8666fe61b681ff50cc4fe7a4e86754d16c17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/7a0b9e83edb694b385b3abfd74310f82c1ed369a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/cabeacd0cc60355cd566b6c912a0d0ce255c9a86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/d71d552540b767d71126986d2f5e8d41b6467dd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/ec08c607c008a82226d5a9bf6058b259790aca9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521188.19/warc/fcf961c9d9fe572a6e59c6d379ecfd5057163840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/4be5a32bc3414da6416e38464f4d31d2d2e7d120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/65d8f69e8ba89889e201e854265e17911408e7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/72d964fd9b395aaa4f8d0b768652be457b43037d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/9d980519ee158f1f8c40887b2c5594a0ca6eeac2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/b14a1a32ec5d8bee392be7f853d0f819213397fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/bb7a88bad710abd6408d4d6f7e473bc8e68bd2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/eb7be72aff4da1288e2cd58daf47bff37cc9c712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948521292.23/warc/fbb1742630b7ae9aaefb1d3e47f4b279fe7b0a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/07b23af100b7987ecb162b1a13ea4369cf7bacc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/07d5d144bc4a0b1c8f760ee38e7f759e5beed984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/367b9e52860e069afa11e848d0134b93ee83edda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/5dd7cdb40642f604aa0fa3fe04657d2aeb719723.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/6e0f00ffeb0210a1f62ec87768184c2668faad31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/7c9edf5f4c962c6c9294d3e12b308f88913cee44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/8ebb89ce5836d283f3bfb192499929805e23b3ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522205.7/warc/c3ec58d8715a8a9d2e686011d3c7c4f25ee62902.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/0a34c701dfc25a816793fc0c3fe54cfd6efc249f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/17b3ce2ee41ce42576381d06493c8b65c277ae6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/a9909f9ef22eefe235bce7e3335d70e024982c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/bb5d708ba74be00ea78e66755946e5c0c5b22574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/ce022045d8cca192774b3ca049464beffc94d982.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/e1e3a66eaea51758f5b1831cdd3d56df5ef5acc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/e32a132780f01d59115a2e7ea33bdd2986f1b40e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522343.41/warc/f7beed8b7613d1f2da793a12def36ef8399a24ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/17cc505b53d7e6cc0857a427346812ed9a2c8ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/19e60858b98037ac7467743a3ee8cd8d9164696b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/3ab04e17cbd98328b3dc3034c9c65fd9aab5b3fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/52feeead23d381d63822b76800bcd705acca422e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/657551dca4be9e3667e96d4980b14197e1b7f1c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/c98c0bb98e6c84513e8d84706e2fd0cca6103a14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/d65cff6e0d3e47f6d6a9d768c3b3e8d5bef10e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948522999.27/warc/f3575787722844a153c9521becc6cfb83cca4f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/065db35153cf51b4af4eb5d6042baffb8e177ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/3d98019f99dda390a199fa35826a73b76a48c5d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/47e8569763c2a63ef39bcb077f7dc83a5b297255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/4a7e279666af0fcf795301a357de55fbbc60f3ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/774969210a4a3adeb769b09dcb0b255852b6c8d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/8f9ee7832859460dd6f097c7151e7e0236793c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/a3f6f4887e7af880ac24cf5ba3be531bf46a4be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948523222.39/warc/e1ad58fc754c644981e56677b124332b82a90527.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/2663861a98e4d32e5c75f5a2e9fefc2a08ce6919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/7865959a932f1708b521940630951297a27f15c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/9cf43223d4ceb018b83308ba7423a0c5ea061f02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/a65f67b42940ee47d8ac19010c508c62bcdd4a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/b0d60f89a5bb086c9c0be81528bc098ef9ce45a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/d03ade8603002413028546a586e18d08e283f136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/db024dee07b018a13909958e212332397154f812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948527279.33/warc/f9984adf0808a092b31097e3ce81204783cb05e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/3028f9c66f61bfa4da4eea28af0b4237b85d24f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/58822d236a7850cde5e7555cca30bfad716bcab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/735ca40e9ee21c9c1e3e7d3d5d72525052951f06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/871888543fa315c30c77f0eaf160e87daa93dc65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/9a9252f22bf69794e06c2b2b8b0d3f48a7956bdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/a467ce802d980dee776af032713a207784fd5fa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/ba4202a22d2acf7125f1f655815c8e552a032fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948529738.38/warc/d3004219a2a2ce4bef64f9cc24b8af4026ef2ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/25cc284863d1fbf6d00710856ba36f1aaef1c10f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/5d16d7a70c9062c8aaa0ad0976b8e0ec4a2f97a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/719c29a5c8f02bef4d238cd61e41f684a43fc2bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/89c1c51dc9655d29a9ab66fc615b3b7884280c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/9aa96c3d8870459b9834d62e85e519f41653a388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/c7d902e8efd43ae35a39dc1e46afa3889d8583c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/e192a444118afcb0af6b901b9ef8f7a416a64ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530668.28/warc/e3d40b440856f11962d195524d966bc761526677.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/4ae30b61a23c7311f93c3b7286b9f2c66c402867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/5e134b5faf41c7227cc58d3014e850df5c52bb29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/67118eca7bc714bc9d38fd227529e278ac53bee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/9cc74144b6d0627e0feb3cbf03eb573065da0b94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/c2746d2d4a597df8933af3e21b43145e97413de6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/c41a4df4a5647d8513b3dbae89d48cbfe7a0f326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/e24084caea4c3dd0b903ce1eab9b511b784ebc54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948530841.24/warc/f7a3b703606b97939eaac8f8e268ee99343211dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/10efcbf5a5dc38bbd9c8de3d1112d7d5370107b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/1ac4086b68199026a511bb1793f52a22474e8140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/1bbe5e59e02f2f8636d16da10060c6e07e3d2d79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/460d38358a3919c15ef151cc17d01ffbdb33298e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/74114460b11d429a5698f9760fa9db946e982515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/8a5e6318fe6c50b58cc6d859f7848b7047911924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/a05a3321f31812380e98f10bb5a280790b37c2c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948531226.26/warc/be50524c6377ec12736088eb9ae77c97bcb0b3d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/12d74f2be05ce99a0a0b677c42f5de019986410e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/71feace8fee7b66c05196c98e71bf5597dc20efd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/75ed3bdf12660f45e6f5704f22bdf4ae7bbad429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/7eb1b071b85485510bb558495d813d1ae6901824.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/823c611cb4853974ff51ab7a60966ffbea87f07d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/8e38b36112486d234ba52e79766af62dc05311c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/dad91e83c81533491a5da6dc61bc6c659de78b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948532873.43/warc/e2b1691cb13280fe28619e70e5a3aa8971323de5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/4af2ed600cbc861f6c5ad14f99a145ef7b4f49b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/6e0231fe4939fb39faf09a1122b7f283b1694f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/875c1e1262684be995898ef43643c75c2ba32f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/8935d0aafcd7e307af45d02fbb35402aa7606651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/8bef74b505ad441d2fb48a1183fd4247de5f77b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/b9430cca4234dd405dab34abb6b14b4fe22c2617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/bb72c9570fa3cb10033c9b8a3da4a4dedce573f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948537139.36/warc/d7607d2f88ffe8a434cd0a775d75cfc56ae9e6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/18afc2fa1fdf391ed4694c6b2636a9b78b121ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/243207a2b24589caae626ae2b9ca9473bbb485d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/30bda0678d90abf760d6d3a8672a8960b51db46f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/465ed7da97172a3ec623433792656ae6a9a36f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/4766c4859e9cbabd83b0672ed00911f2e0fd3b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/5df5e1769ed26aef5b766339ce6396ec631786b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/76f6f29b14aae33cf4d532f9162c2718f8ed0534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948539745.30/warc/e6e678e4e6512bbd98119b5b8fc670ff07883cc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/183b6bfb1bb13abc35525e6551870ee090a5eb83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/1c61ab68ec656fc834f781b6b8cc0e554557dbf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/34be67202bfb34aa168a11130a89f7eea3522356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/a49aac3b043353cc2e3c9176207415f47f227c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/c6767f173eb2722d27002d6c713495183f547ebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/d50150d011409c055cca0745f23a06244071eb0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/d94020fc15640d32dbee3a4cb3b2eac9fcc525a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948541253.29/warc/f9da0dfef15be6acedd9d799b15669dd82ae7014.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/0dcf1fd408c483c4da808b7af69ac74ae3549589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/1aa9c5cd2e93df0f054a5ec0bc283d958ad9333e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/3fd191a4477d2d1f220e28fd6f80d383f7fdfbf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/8267d7b7c592b4806e4e92cd567131031c3d3e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/a306b747ad7228f08c15567aec73ffa54a2a5215.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/a49c5e3baa928ed4b3d14182abfdd4e407117b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/a78c7eed36464bfbebb65eb81d6f00ccba430622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948542031.37/warc/de8257bc342e65fe4d9ca4188966141fe67f4287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/16cc704555c452af84199167be0764d13594ed7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/2c567975bc6c6ec1912f63a8285f66079f536377.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/53e069cbb14cc6a09fd47bfa4efb1a6a38c88739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/8104f4fc17813ea99af7845e7859077f9d1a3809.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/9b4dac0cec340ad9ab0ea1ae03ecfae58989d4a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/ab62f49988c15f032468651a225132c23ea3e5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/b6b387916a8d75294ca9ed95f4b40abe6e9534d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948543611.44/warc/dd855d34236db52009f239564a3b769207a0c654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/24d0898d9db8ea2555ac50d638ad80e9c6fc8faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/6c3b48be8c636ee35eb269a7bf07a1579875350b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/8b09806c638f3ec711bb524a7381775f98cc214b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/ba0e1837114a241512c0662aaff28e2b9d0c0e0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/db4ed40cb297bf38f69ef427376345841ec8bda3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/dfa979ab02e8fc3f3a8eedc49b27335f8ca71aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/e1ad44a58d78321232f25ab382d830ad249b8bb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544124.40/warc/e1f9b79488696fcc3392e958d2044945961f7525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/02507bd05c27c3815bef90f6ed04dd4c1f6191d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/2dde1d62dd9ffca8c3d30e120c4aaeb8e6b950ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/42e6e57e1111fe6a8e0924f9212eb0e15b26821e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/60482f38e5d6c5bafc6d473f783ec965e5433267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/63a0a267c0287d237c8dbb820761dd9dde1b1dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/a2de9eb8e8c9dca65ef8bb775fbaf2826ef1f000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/d77b06392beacea854211c3d88a9b4d91aed8260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948544677.45/warc/f416d41123b13e82dbeb908a66e0912207f578cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/266ebff53d58fa4af85abdf89977d389d6406ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/71f35997b57d02f49f095715795ce08860a2c278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/78752e60cfc72de12153e4ef66c2bd70eaf4eb4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/7ed817447e59539c0a23c1d5436bb695c52d02ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/aaae1a5782adfd63e1481d924af8c47991702f50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/b376e0b8ab44a488588d0941782d367c2435e9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/d9722f30deadb417a870f9d6f21f7b733af80816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948545526.42/warc/f08c2641aac7bbcfda3f7f342cf009753bf77e91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/15689602555c873a30c51b8f85ce8f74e989b243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/30f567f7503ee32caf7e22a8c5571222d4fcef95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/50da96ae9cdd8fee5ab8f74d05e61260617d260a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/6ff79883c9252e18241c3254b468efbc073779e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/922e9f6ae1586688783d0d153deb7efd5bb18cf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/c66e331933280ccd6ea26b023a73afb03b4b3095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/d1302fa9c289acd736a2452c7762177962850911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550199.46/warc/eb8b811497497beada180ff7fbf639ae32943e96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948550986.47/warc/1a9f50e7c425b42d7ddbc3f9d55a2391600b458a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948551501.53/warc/2018bd5e4d33b2fa84b8eb18065ae92f50fff40a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948551501.53/warc/9c71a3ecb3f2445ef0610b5ae88be4474878e25a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948567042.50/warc/8adc04ac5e942134f59d192eef6e3d44808d903a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948568283.66/warc/6bcb7556c5fd4ecdf88a7daced086d98132cf3ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948568283.66/warc/9a50024e07e01c8c3bffe8d59b4fad5f8fed18f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948587496.62/warc/99a3e746eb8de25163ba9f3f0024ad9643c48e7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/1512948588072.75/warc/c2b8464642e33b8113d920c9e8527103668d47c6.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2017-51 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2017-51/mix/input" + output: "/tmp/cccc-CC-MAIN-2017-51/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2018-13.yaml b/configs/cccc/mixer/CC-MAIN-2018-13.yaml new file mode 100644 index 00000000..4e38ae0c --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2018-13.yaml @@ -0,0 +1,914 @@ + +streams: + - name: cccc-CC-MAIN-2018-13 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/0699848c4a7a55492bdb0da35685f8905a292ebd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/09956a2fa53ce7fca30545bf4deedef1bc4b04d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/74faaec471627838c0e4d3d7834cf81e0629fd56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/7565faf8ee9626b3c620bbcf1c3b3697e98b9ba1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/acfcb244e02578b46ecfa0957f9ec21592272347.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/d86e8475b1d159b294f8840d0ce7175a6fc414ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/f35b77bfa4cd18dacb263126ade1f57443304eef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644271.19/warc/f527fa0cb0d806b6fb1118353bd292d374db4f11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/26efc8e9ba19e67571d94197934ea5f92090c695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/50e5002191862f668836cde9dec262e8709f5a0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/53cceb1a96f4c102b91abaa45fe5e6165fd49334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/99297659bf11caa34ec10dd71aaa3cd485a38163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/9f5bc9bb840501278508c075d45dd3e67c52a690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/bcf43b501c701528f23af3d22ae66d976da6957d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/fb458c14b5d00eadca2246e543aa4eea7f991a7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644701.7/warc/fd69933e3030d7049fa98644426c69be7e9b7b0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/1e3708154a7245d70fd121c360fa5c82d6f50849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/2fb00746e65e9719cdc1c4ee3d477c7ba528c55f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/6ce19b1c8a1727ad27e4da48b59c75a17aba924e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/7d3c8311a5956bf08b4f42aaf1dd3532a7f52961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/b51e5156d1dce7306577ff1c716fd399d96410e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/d58686976c14644d7917214eebbb990029306785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/de364cc3fd7b302a881c0e94115c35674350b251.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257644877.27/warc/df164a106f3fd6dc49d43e15d6ac0acba9f924c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/4b42fb781bcf59acd9e859d6f16a7e328edc433c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/7ad195d109528de54b27aed5cc950c9e22ee0fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/84c5795ac20b60e2baf77ca7149ab60eba2654fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/a72ebe9eb27c9e3a2edbdd0f8ee56a75893984d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/a9b345f68b614a036ffede01d65df09b293c423c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/b0a92c0ed3cbf7142fc1093d2afbb953a4d30b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/ddd40db34e2a027694646fe19d14fefef17f13fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645069.15/warc/eeda7ee055825be9b1816d71dec8bd4494b6f5f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/0fb7763d2cf8e4a0eeb0f28f81f3d224997322e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/178ff42d00f57bd6318b879522e4942e2914b360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/2e5e1fbc8b52e4b985f89dc54b4465861b015dbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/6dd2f7f4616a3131a683ec9a8a0c9ba80fc4c93e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/c18be88a4c8c8755473de70789ddd42db602ada1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/db583eade76f2dad511b680c4c81a6cf962bb86f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/e4cc8a433821ff9a3a6a1472eeb30d7afde5ae1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645177.12/warc/e57f64db9e4cf7fe08fd5ecf638e5e34947462b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/09a60662a0e1ee47f2da8a626971ea6a2b97b0fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/21f552e85ef11c63651c85578592fcb63d76f734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/62fb826b2a28b332d3689c21994e497f65347c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/63760168ee27e68696b7401bf62e81de2e474538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/9cd9c46f4a6bedb42faf530d888e1824820dfbff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/bcf944c8c8b6e1269006c79b81f09cf8efbc3057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/c5210e9683ecdfaa967caeef32bad804abece54e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645248.22/warc/ca05ab178303f926cc4949f99c97e0cd41cb5ca5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/0ca3819218bb7e481bd2ccbb840c42d707cc3579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/263cda93cc0149f10b65546cb134dd74eefcf103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/28732717bc8e1d2eb89adf16f1f3c37d430b6ba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/36b25e59de0cf471d1627dca6943a3a60c4a53e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/415ddfb83ff455f8d13966d08a240656954f421e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/819d49b32791a94705f11e097265ef9700071919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/846b4646af5231c48849140048f56193e4e4140a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645280.4/warc/c538814f43be13a2cfb2b228859685f5095d0d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/1b82d3a8b2f8501a0a01e66a6d931da8607a16df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/26f76b3423162f5784665375fdac376782a5877e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/2e982eaa8eda906c0bf99bc6c31274e51d3150da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/45bd0dca90d09fb40d3087bdc8f7143f2326ac09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/45cba924863d064ed5c15eb473cb59a0c2ad4f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/87d09f62d842304bedf6c873ba1fb440977ca8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/f00fdab3af6082f407d8a5a05594ff1a31c937ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645310.34/warc/f6161e1feef69b40952601523e7d378d0cf2a060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/210138e433a06a5d33df8d0fd1810457254b9dbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/25a38ffc1db1b8c1ff3539e77498e55619cf38fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/3e2f695f308dae2468b003dcff3d4b74f106813d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/b84d49a6d583f783573f399e5e7288cba2497c97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/e030505de42129907b23b62fc60c09a7f3eceb59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/e12be8b00ffa23c76e5398f45e2d78311050f3ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/e7157c21425b6f0009a867413db7b9bebb883073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645362.1/warc/ea0f0b32976f3d176df17a891aac8197b692fe50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/0027fbe975754404801c3a43c2e0108462e05b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/180a874b8736a06ead31af0e94cf60868c0992dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/35c14830244fe7a30c12c11b7af5af1007d29f3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/4b4c588e35a36a9da83b8148c6aa09ccdc2cc310.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/5acf7e532b8fe1eb36be1dfac922bd6c7f1b9d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/79f378ab64166b2f5371706d6760f5f33972bebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/7c4237869dc2e36a88e0f23529f420c09c6af6f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645405.20/warc/9783ff5cd04562a1f9ea572e5244168cdfcd92c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/16e2e67480188576216b17e1513354ba327f2627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/460601433fc67cac5770c437c98c22b2c28474e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/497967ddd749226072158dcc7993b505cf773a1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/4a65692d0432ad704542094aeccf55aa13b1cc28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/89e7c45c80ebbcb68c85f2d569a777011ac00230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/8f587722a5977a86f5a02673a95975ef99c2c756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/aaf04987616081778db98d3f38e97ccf2c3b6c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645413.2/warc/dbce2759253db1a074ec7fa934d947d26d491cfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/2bb24e72ce993c7b9c32f1c42a44af25a1c3409f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/4136bf85c30b21b26ea5b3b15425f39db9d3a07e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/4807e21b9d1d380dd537be4959819f21c27461cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/7fa500401d9e04e1ad82e0716f959ab55dbfc7e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/8679ea8c8f0dbbca7352aa0fdc1f41bf9a95d85e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/c56ba42e7c27392f8c5ef783b7dd452d2fa03c5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/d1adadfdf327cf075b63de04c0de31b0f317daab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645513.14/warc/eac7137d97bf8cd8265dc626634fae3b2dd605bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/0d21143c135f03bfdbba0c094e83b78d62f36fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/188166ce60d49b9226bc8aa3dd99585cee87b448.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/2195d2b442312fa3e39c0d956eaeb6d3b0c712cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/47e35f4ac164afefa4921a9b90f62b72f09df3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/6e93eebb860b9b19c1e3af05030c159c14729a3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/c6f0c6d6d9b8c50cf87febd8f4fd4f8a0c1677ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/d37fbb1dd795212512bd11f806b030c5b996b1f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645538.8/warc/f3e0dac452404636353212c7d0758ec8246736ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/1325283029129764fbcb3497db8b4c212a7a44dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/2ef9b534362bf1ea4ad574d50197a37f97acb067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/5408b489c329656f3e12ddf4e1cc4a814dea16eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/60492a6fd62b8f8fd20b03eada20c5d9d6f89825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/67493d63976d79d8cdc4813634f477de461cf2f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/a96445fd6d49be29ce48399926347b279f0bc456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/c2e1235f2b48da014360d3dccbe359c90f312d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645550.13/warc/da2353aaeec7e6b993396d01e278de6828bce764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/01d473bc30109f2e6fa62bee7db590cbe5fcb10b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/054dd436c4812bd4afd272cb16aec4a183a18a80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/54f7d1108c7fe0d8346ad434250e2feff353fbbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/5f9bb2f32a1ecd5bd6c795d33ddb79d7907196ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/68d1ea469e62bd2808eda725b415553d014827a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/be3fbbae40ca86e2882a202a8390302cd5d80362.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/f29cb36923c788e0589696282e8485358e775725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645604.18/warc/fc746f42a9efb6c2afd73641c8477ad7480d0905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/07ecb5f73c57d1619eadb6617ae31a2b379c05cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/182cc6046c7c49e51c89d722d4f8ed0b1d9c49a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/1ff327429f44d74a7e4dd56fe73305425239706a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/27f9e743ce6620c5338a793afbf82dc5fefc188d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/627b58446f2ebc4ff7914baa697fee20e7f44e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/92c1b2d2ddb529f9e1318b51621538e7fb7d3718.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/ab0fc83518f9d2cd170b7363d6a3d5f017e7212e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645613.9/warc/c66b463f45c4a660533dff3a477d40b0fd6e7ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/1991d91c231727dd6ec9b9283fc75d1a82343c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/254c1df62687d0fbec4e864aebc825e32b0424ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/32c5b82b42264cd63686cc3e94ddb4a6046a233e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/4e886ae84221f6692d4c9ad1ad86650171c23145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/6ed9f6e3b37b75b7fcffe133085ce917c8881e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/791bf0c1c1854363c61e594c6975e2b4817dbec8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/a503187b745092ec555e27b44e97279bd12600d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645775.16/warc/bed5e9ace179c6a49e0afed6d209f95fac2d540e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/072c7928f6f4584d6ddf556b617993dc4a4f0e0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/09f8719edec79c4890619043ab1368cf87ac470e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/416a3efb67f3997658ad7ccbbc02474f8e4bcb36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/5018ac2a7da267987d5e4d7ad2fd7abeaa9b3e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/513bcf66f193c9be058316ab9fa80abcdb64a0fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/60e9ddee0e4c80886140859693816b56aec737e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/657d5392dca3919b3580e698441323437ae1087c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645824.5/warc/84a7714aca7152ab1f92c56f32d0ea3bbd2bd185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/01b9aa44b3ac4285e2f407df812bc8bc0e992673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/1397958f64939606f1ef8f6bf5da00571166a231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/150eab2a95f52a1d1c089baa0d6aa8230de7c789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/18a37c60072fdbab9fc8881146f70c0627f802d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/1d1c6c75641df5ae34af73f571a34e9cb1bbab58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/61c757eddc6cc72e00215461287d9e2d36f5ddf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/c5087e7328545486d3883b851e658901029cfad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645830.10/warc/dcbba8035d9546c6f17b685e9de713e3e56ca860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/0edd04523d8d439544a9d1ea92f7b3585dad7c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/347342863799908550a7afd41761047a8208df0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/3f5562b843dcb3022f790e66bac029dcc2d9363d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/4b4cef8637091318223eba8208849ff7c522b651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/c0aeab8ecf119fc767e2b34d4d4a2e1027d2158b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/d75219e2dbaa9d2f16cb7d64591f408dbbd0c2a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/da15659dcd06d6307981b6a4fa963425a70b53bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257645943.23/warc/ed68ed58b9029f0115603a9d6f94e4dfd5d5f260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/0c5e87c753b5d840cdcbbf4507817e3c60dbc0ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/25221316a55e9c3198e5435d792d342f27fcbd4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/53d7d7c7732067113334e8c178c8e00683288fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/827ad910dd16082504f4e6f1798cd33ca3b71d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/87fbcce2cf990cd49049b76474ce702ca6079677.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/8c93627a0095befbeca2e4440ce3a4753b8ab3fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/cda69859648df0c7c42ceb634c54111956e5ec88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646176.6/warc/f95f087ad2e7a8fbd6e3f5ca9436dbd266ccdcad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/29c879aa6215d2323f3ee31a437531d9ce0af357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/2afac65a9d8a97b64db96f59c9912680f3f60616.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/4cd850f8646205614cd98493fd785eed30988266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/4e781a96bfdcd521da8c7f33f7a741d4679741a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/5238bc7576c79bf10dff39772ab294ff6aa34473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/7aa2bd3393902cc56b59bda41e4e35ed586e8be3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/b319f2782dca559b99b049233b2372ea989d2024.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646178.24/warc/dd09725416c40f21ac5ea96d1c8532df3b05e507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/24d0b8f9eeb08f87d138814939f17caaa38366b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/27e97c372a54ebd246d3988026bbc9e3d80c5ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/28458ad28ef9207a9f0c23e942b945695aff6d59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/7bd6e775002fa8800a17939e1084308c82fc4af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/839161e29c404373505ed789de956283565d0151.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/99751d9211b674599cc54499a73c5125393f5760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/db9fe628e0fa8a57e3fa3ba3138694d220ac5ac9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646189.21/warc/e665531c6bfd8be0d77a8f7d313a0cfb2abc69d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/0115d6c2d1cca8dcd078b2f62f339315d1405a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/aa8c7997af552f2d6ccb77288d6f5f3ef429f44a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/ae8465bad447e8a5cd52099208f5ff223c8705c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/bb2121c9dc6fbfbbb394e410af78c3c7365e6b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/c5a5fcab3e2a04c35d165966300063adc522c0d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/de1de157d7af3cc10ab0ebf46c97e176f81b5405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/f74a5dd9a9c60464befadac1cd9475ddfdbe1c22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646213.26/warc/fdf9674067547699e2380351d8c707f486699cd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/1938728e6a6fcf3b97a107743fd47abe5b5bf03d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/252c9806e54d1533311f00999760729054b21a79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/4912bb402e41f8536d33e9098e759fce74e21e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/5df30697fb3e7bcb8c38a9941ab3f44e60345b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/bd423e22b0efa4acbbe9ce5120c71167cce8f051.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/d8873e0057b9e39f512c2882074382fa9712d647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/da924de22a3e063af098f629a7894eb50ba62d24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646375.29/warc/f4a9a26b1324f77232e823a7dae830c56ee3a4ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/3cab02edf2d5769b59d0a0957414795754fb2881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/74545cf3d39f2d4a3ebebcc8bf4c7abefcd3c684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/80f9e392e27b7dc0c88f04d09832cf32ffb822a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/cadcb603664b6b2e3e251a43931544dfb47217bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/e406d4e03f196ef4ef0824dead2ea5927c811046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/e8a1afcd586c6c59e43dba8793e38719a8536da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/f42e47107275a761d0e3f6122fa5305fe5c73519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646602.39/warc/fc126e7989d3006cb625b871d0e5cfe9346a0155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/25f6e64bba81ed11476bcbe88a1c3db21b1daae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/39598d12ab0f3019c07bc21f4c201c4cea09c1f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/586194c5a1ecb48e4000a26d1ad1d7e4244afb70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/60853caa892a4f439a78fd4c1192ab1d55d71542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/798cf7586f2f4a5b3b3d1024e5066ef49b86128d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/85918619ea7fe6232e8decb731635647c453701a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/9cfc0c1ad11d0f8e0e17354bb4a22e9ebb70742f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646636.25/warc/ebe5e4dece1f77a6ec725d2aabb607619533686d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/2b5b528ce237a74b33e49a562fe9c286832f9b5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/320b5dd8923d2cd700ae6f8e923f8c2733d01a8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/36a8a3d9bb8e65ee6681ab7de173c0beba305e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/3f856f367e7f6aba29da65b74d5c6c608b3fe9a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/4de364aad6f6ef6b2156a9a934f9371d5359bd59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/99ee5977e9e9c1750ed7701b0a39cac7cedace73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/daaab94174e93d7974e35dc716c50c67e03d755d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646875.28/warc/fe3ca8a700a1a1bdc97f40c8f13c4ea5bea86199.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/4ba8b69ba08202b76af17931e882c4f55567fbcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/5fb2bdbd624909a50e376b095c4798014f867b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/884a358af41a05287b58ed087e5beba9bc5dd166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/944d3aabd7a26b203d2bd13a1bb95d4f959f7bba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/94f1c056d0c1b76957b358c5d99f7fe6a772e76d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/a2d5205564783054028045ffa73d4d6bfa084e2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/dcab204accef2507953d99d684cd30c91c91030a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646914.32/warc/e03b24a8b16a9fc1f7e0834b4818d7c71a5842e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/048d4ddd84c6d7cd90f8d4fdbbedf9b99f91fca6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/2d7aab2f4f004351716f55ded9f009d67a3ae017.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/3bca61f46bd3fa5a581cabdbccb3e516ce49cb8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/613636a0c8444697a1f119e5d2411136365b0d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/880eced9ec9c898379486e009d4007d351e66aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/9b03ad0146597711e3e27621cb4444f923656f4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/9d82ee9e25dd6b73206cdc354f02cc78b170862c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257646952.38/warc/d65127b709a8c89611beaa9ae7511f8b688e2b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/0c89665bca6087ec7033d101f22a8d635176a85f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/19b7a9fe67b112db7a5c16b1fd6743c774a992fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/2e6857058f0bf6301c266ee156daf35e69ce6538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/8aa2a77cedeaf69f16c7bc2f98d8bbcc3bfb90d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/99e7072bf53fc0495bafcd9582496966b947ad8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/a094f9c7d184af1478be0aad5c859a6ef4ee8752.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/b6ba06b0ed24c2ef8666abd16f5ebb6049fb36f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647003.0/warc/f63ce5c431d15337f85588d585f8f1f2a5bfa48e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/1acde186b571a9409454f47a9cd4c6f5f4ca775f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/5b81c3af30d1ce9bcf4f22ee21434748aacc9c14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/61998aa8bf5dcdf3d1fb695cc1320f020787c8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/acfc5e9e60a90ca18939fbb2d2ff3663f3000e58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/bb2b81a85560c0c69ef8b93490a4f8af5650e404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/c30f82637d8cab7d766cf3293ec3e68ab3834825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/c824bbde7e9dfd99479e3d862b707d39717e64b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647044.86/warc/e0e5a155bfb20d08f44d8a39f6ca1dfcbd982855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/360f3418ea16a8c019f28566115bf19d86d8a404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/5cba37f0e44d7dc43d0c9ce28105bcc0b4b5aa0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/6ef2ed214d6bd5034ce6c115dca409d86eccadda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/aa447f256bd8eff6b5a18c6c235c1ac7c01f9cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/c9e228491f1355ac25637697d3d936b73121ef4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/d3e27fc823c6a936a6f459d1ad76c9539dba4005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/d6ad90c047316327ed18e55469e9f164da131531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647146.41/warc/f79807a268c1bc2f40b742689e0ea77bf5c678d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/0b7b8200b1aa1322079d951fa0c3b0897406009b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/3047666c512bfd246d5fcc77213a994cf3f5ec93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/30dac4567abb2215a213203319d7fb6bb878a4f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/37701a40da87d243aaf1d8c6c76944e854309cf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/7c5efb9dea3b68271d7b785771f44232549fdfe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/e236ccdc92c5775114f3f3b5d2217ee164b97a1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/e696b71c0b2e3be3442376c692156b472cc59dc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647153.50/warc/eb45b561b8b866abad5790ad8b784275b76a60c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/2f4cf803b0dc7696f66b2769804b19e8feb539b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/4ed71f9a6181a188c710e526b9ae1947e1e6ef16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/4f1915617914185f118431609dce494deb75a6e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/6ab2c11f40b0155594a202d83e3a7a586a2c1c51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/710882a471623eb84b41bee97ee7289962b6c8c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/b01a34df2d2089aa1f55351dd82b3c494dad3022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/d375531df08cf1e0a81df7813b9b44a4ca937267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647244.44/warc/dea2950cca7dda4a1bf123fca654ea01f8dd6b6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/1d63cccdc69fe157f7af6254b00b5f6463ea80f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/238cae637e102d6aa7918a0b45f36f0daa71cab5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/26929a67b96a80635eb8009cd5e6e73070638f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/28bd8deafbdff184bf6e82fb775e68399332fb04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/7eecb150c6439099873099ba2dd642b063f97781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/88b3206891e84fd787c18572ba2e808026875a5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/e2f938db0a758c8b86730bb95108704d48480d50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647251.74/warc/f26dcf3c241fafdc5af76e2af72741ecf5473f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/01c94b00d198ae42b5b7c5bdd35dab5ef3d0e222.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/7bfb23617d2626ad1b864779eecb56c155baa021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/826df93a733755de903c67eb0a6ba46e0b4197a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/9b50e406efa23e39453368168fd07b75b899616d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/9eb8a90c11ac58b4f792a804c351a24620bb601c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/ab179e504e616a7ed6e73baba597ca1031903376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/ae27e724f8b2e5d66acca417829b22951280fd0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647280.40/warc/c754769acbd0a2ea99c9e07d17039a3838391e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/0576a6ceb47c5114a7ad8c34e628f768160239fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/0c4f8b1e1ca85ada5f738fb12b110e037b6094a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/0e7511738168d3a8970adbf8897b79654eac3675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/417eb8d59ad67a05c4aa9dfab8183df2a6403227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/6455e0895417eb124a34d9a17f008aeeb114e6e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/7bc9083403e88fc8290ac1b60aea17ad41c1f8a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/ba784446d56479c7d461763c5f2d7f012b5f283e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647299.37/warc/f69f64aa3a1ec0d94ff1250437004234690e0e4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/2924932779c6a2f041f1503a0669706108cbfbd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/5f6699e093bb2b5764086acc781e7756e8b3f979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/9d3331af406257a416fcacf8d38211b3ffdc6e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/c26648e54b6a0c3eed64d1eb7c98ed8d9f0bbc31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/c27f1323ed13287af01afffaaba1682655c9b8ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/c66e120e5b1f39743a9c2f2a4573198e7fa00378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/d8cab4bf6773894d03210955aa8a69f2ccb03b13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647322.47/warc/ee4574c1de6efc25b083d9ca059920880c73eb9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/3514725a7f9add2bbb671ce1b60bd53694ad20d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/49e23bef89562636e54c89904580e035360cab39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/4d2b506823339e843d65c23f7a93a4ef242da502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/914caf6463bf15a75753fe6f75614e3bcafeaded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/b3fba441edbb166c39eb2bcd5abd15262a0fb12d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/d1c5d7ea3649fd89bbb8093b402d3d317a892e92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/da9bd219057e8d3bfca7c293af22678c19201254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647327.52/warc/db2b89c5d1570606da56fdcf0adb81af2f147b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/09c53d640c08fa5d538fb7975165adfa3d83d0f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/1f133fb354ba958640dd9992ed1d658e068e8bcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/255000638016cbfc3887b37eead047789c07fc4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/42ac7c41ef29a593e2ede9cffff9d05bc17196af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/4b631389d29bbd0bc1d359c0c35c234d08818c1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/6523f6e0f8b5258688122edb3927a5c14411a19e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/987a7e48b8468a89eef0e95f6ddefbe2f445e5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647406.46/warc/eed995415c86078e425217d261aec8b56898c7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/0d0bf1aa9017fb9bbab969539b8435ac6ac91b43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/1aa6adde0eb3df9b2571984680a3537c4d00ed93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/231a1c76f7d2ed8d53603c702d5a39b457012735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/48d81f5dcc2827baa3de1051c989c6e3e15a0dce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/5ff1c76db345acb07f9fb51b96e9d3835f60d7d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/8257c979d1af2ff4c6811cad66d82338d56d84ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/9899bdbb64d08fe1f5d3883ddcc5ea4b741b81d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647475.66/warc/b2dadd93a0dc694d08770db8963547bb93ae5d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/1ecb12426db89f8e769f311463d339100e0bf134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/3900bc925d2ae5ef48320d8b410ec21418098322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/791a605404a79a505c74852fc4c022a79e575bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/8467b9180e0a1bef70eb2e486c074346f5cb6e08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/959077e3e3749ecc88e654a4295b16ad467fa046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/9a42383478f99c192f3ac0116ce6e640bdffc1bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/e394786cf94b7cfc3964368d444dcb1ce2f6d96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647498.68/warc/fa2b389dd5866b6e288647472138aebc6154e2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/23c2b23ad1322a9bbbe92e6ff27314fcd87af013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/4aed9b3a519836700d50395556a3cc730670b214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/51e5c01c8997ea30d01fba81fc3af098ea359412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/5739ee9a3c8006c062c3de14352a6660a307c53c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/5ef1866c01e0066b1acf5a1ac0a3698abb3ec383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/69580315a495fcf51e83e0c32754904008c4f8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/da63bd893aa5d340f712059883d048687a220c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647519.62/warc/e200c70491836dd755639dfd6b5653ffa637fa41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/02419d29ae71c1f34ed86a46ab16340208c09514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/1b4fffd580f316e727424feba043e87973111157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/560bfddac29f643458443c5de6551d2591ac8f2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/676339dd291be780cfd0e2190b8e445d93f46b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/7b6127fbcc64ca0f01bc61259bfcc5d9b0e59da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/b6144a71a2973929e78f91827e376373351c3709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/e6b4841afba0d4e1decb0084be5d4a48ccfb8726.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647530.92/warc/f624d86a62b84fcbb9703e06e962ce4c6bdc3bfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/547f282ec3814acfe194189cdc8cad4dc39df209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/71a82dbd2fbb070546c4639b4916e49365341622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/754fd63b3b937ec8056ca5152efee293c4324d22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/7d9191b7803befa8db4140706473c38a510ac449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/8afbec9ad46f55e5faad810417707b091562f8e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/af43dcf45f1c74b7aff774c7bc09b618bfd5f69c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/b9a1ab07a3dd35bde7e1bbb288170211c4da437e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.54/warc/c94268fa5f6ac0ad2b56402f1b84885007c7d1d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/37f8079e9560b8c72dad26ade21dabad8eb4c971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/46cfdd869969454434aafe72d3d3ea99acb0f567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/76f03dba9934796ce0b762d97bd6b2230da7b0d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/87aab1fac5fb250f0ec3b7a3d6bcf4d9df763e89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/bb6840df7201abeddb6e49bd303210fe4958db28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/c64944dce0b94f1630725e0121487d9bc1f47d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/feb410245327b3e628776de7daa71a986c1dc494.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647545.84/warc/fee52595e686ba5bbe3a633518483743e8c7e6ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/29852762f5778f51b8353dacd871f239c9d9dee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/3d6f55798e24b3469d4db449bea0b0193dc4bfa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/9ea64da1ab537b5967082bff047d7ef76f11cf90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/a5b79a7d57f8be69c325f335bf89ec3ac72dc5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/b5c3ac347d285bbef2b0ab4794d180609b6ed9d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/c38140c6f47f1ad361f49b2511446550fd3a09d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/d1d427789574cbb9ca731fa84729f7a8a78650c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647556.43/warc/ecf01297feaa15ba189a8c3b0e4ee15ee5fee31d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/0fe4c3eeecf3a64828ff57a43954d00e0696e70a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/1281e9b754dea0fcf35fc4f73ccab6f740671ebd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/13ecb7cdf630733e64810d0b9dd90f8038329754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/14228e02719c6062cea9b6bcaf06bf673be2b80a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/29887ef2141e227f59f7f3c682d37acd7f85bad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/33c86200f842f78df5481602949cbff9dd1f0e1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/6ff0c1fa143f80bf4ca56de645064fb54463f840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647567.36/warc/e40c17566308f03aecee8adf865bf0fe7f16e096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/29be6f943cf0781cf1c7266cdcf36053a469b1d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/4a3083211fc99facfc1806f4d4cfbd1a8ef5b80c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/579541b90d0e33674ace96b0f0eade8987d7545c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/7f0cd0ab582c348076827f16602b69690e2a77eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/9e2935ad7fd6252219018473085054e121ea320d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/b54ddb3d38cc6abaf60806bbb5697240802b5840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/c8a54c5eccf6fce3996a0ae2e9ad2f14a96fc4ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647576.75/warc/f3e3df5cb18f0ced637a7ba599de01da5cead08d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/033c5f285b569df97726b77afd0b895a2da54ca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/205690a749ed39e748bb6aeb172cf785300d7c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/59afa2b7dcd6aaceb95a0670df0f089d706459ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/5a2a74d0c9d294776665e1b0b890948dc9c3f665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/6a68a06d72aa34da43501f0e4407c6108bd9beb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/a022a7e2b807868c70bfdb15d91b86ecb769417e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/ad5551e7d134317e95fb0cfb91bb6b3b00f2fa61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647584.56/warc/e1e3e072737d1bc1bcc9e53196482fdc97ede34d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/58fc2fac2a36fd59706982aa951fc733d957dc9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/5e8a3e16ea634ddc2439b99211f7705824709891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/6939d4d0b4c719d07f4951ac262f5081b52d47ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/7273f6900d31abd496ce08cdc012112426589cbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/a71b62bc6f84eab96a7b8037bca6ba16bdcc6528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/b5ca6a6a703afc4870ad890702facc970bc4efb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/c3cff906792cf30e66fb88158597fba58d6632e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647600.49/warc/eda0c9effde0a0dd4b08759a228a80b10c0ce565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/2d37f17b7947367d09e2345c9b94907b9c77d52d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/30b2f12bd39e99b642780910559231c5fca2136a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/6ec3162db23e9d2ba955f3332abb84765c4b8d39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/78ee771ca953a778f5f11a4285af3f9a860471ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/ad9e13cd31ee8950ff83c697d8162c6dcf616c4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/de4db09982e96585b9ae6e9791626155cfe3c1fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/ed0100be73a5d4c05a57eb680401be68020e793a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647612.53/warc/ff47eb0909fbfd8471790fb745c3b03e32d8b5ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/3626ded89769e8887bb3e2aba3e32513b202f185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/62b7dd9438e77991cde9fb24574affb976092d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/66361e646488c9ab9586e0967e791df8e806442d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/7a1860d3f2d9a519caff7e254edde549a52d3f1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/8366f53f104dbd63992883d9b4a47d2d6c9e5e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/a360262ef33cd0fbc5cb0373a276cff111250487.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/e6d26b64cffd2f342bfd8daf380e7799e615a29d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647649.70/warc/fd8596a6ef5ee545c7bda0ec9983e6dbd3bcbcef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/2b15631bf942ea3d6c245f247efbc73c2512b602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/34fb54fc958b47293907b2cc643d1c89f38fcce1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/64353cdf464ff5d096187ea9b4a89bcc3acec5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/6ace3864b036eb09c4795940cbc175e650aca571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/70e061cde1227f5a7f476d399bf8ef4489d037bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/ad1a578b970664ede87e2b3e5237de652a87a493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/b1f8ab8d9aa36b157bf34ea45e743a82e36f2a71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647660.83/warc/dc94ef83837e0fc3eafd2ad3a38e26e96e299c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/0190322ed03b961a597ccf0a978d24a1b995d04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/449c9d4b6c5838c55bea285802ed384e670fc0e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/4783d2c85c9cbda710b3bcabacbb4bff671426bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/5d01cd2ece835ed6e2a5723b4861e16ed1fc46eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/5d377d4704804840a9e74020846f2f3193ac2cf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/9f90cd1e85c65363a0f2349e2b72513780f8e759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/be580afa3e010aaded5242004fc23f808eca83d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647671.73/warc/fc7f51973bb5de565c64e7ba13e17a1af92d9da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/3706d0d24130006a2eb72ce076862894c402f73b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/66f424a81eab267b36f756330d6d58db1c244209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/7a0946ff5260bb3a87ef565cfdcd1916ce026570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/80327557bcaad765a1ea858bd1de9dfdab301c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/978e1496848a186cba502d67e99c5edbc81169bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/c0f6a0a9d65f3d5319066b363cfcbcea812ffd09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/c4f692d17e6d088adf07c4d9914e181cba11a062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647681.81/warc/f8d01116a286376be99e91e60ca110392e351274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/150a82ca14ba74e4556073ae87569c0e657a69b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/56ef5e31a1d1aa98e624b0d808bbc561fa35a211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/70a7e6fb89d116cc387bf3dc29908a5214bfaf54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/90406c3625a811c1bffd707b2d5dacb4684072ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/9ab7639c9126451671d0008f928e1521b45d67db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/abbe56fb76b8752874621c6c93d46284a3aa2bb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/dad4b43cb0e72a6cb426b1ed804f87648daa0225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647692.51/warc/f3377295d4b0238d4a0af21fc0f5a1cfee87ab3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/2ab94110af4564fddd0910649d7bfc374e7d7ab5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/3ae0361191f59429e840b1442b5112c44941284f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/5ca2757abe7ace830a060f8373b8a254a69d809f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/7944f8294977e5d89645eefe0c304dbc05877893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/89fbf75465d686a044daffb09dc3a7108fad0285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/ae5e7089269e5e15e54d4aff7d14317732823bf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/b879b6b6bc144e176a8155b1e1628b9e04928e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647706.69/warc/fe10ffa5fb753f8d55b87a7ee4065483e5c6288a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/0e60f2f424225e397ae56aaa83cc07632153537c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/1f4891fbad956c4c44d9029a85bdb5591974e947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/618e9963062f361ced6d0915fedea35a9a4304c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/73e06815ab61eb1908d62b2c4f2b19d7aa7fe578.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/903d7a3160a4e1b295a125e45d37b3c7e4cb3828.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/cfb0379d99142b7eee9d2d0c6540ac5d1f91cd87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/e43442907e113c31a92b5973f57534d805caef2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647707.33/warc/e91b50ffec8229ad1e1a6c689dfe7832a9903f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/10647bf6c8bbb27d1513c511e311da0834d84f97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/1f9a27a22ec08db39f528a825274932f064b89bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/24eb6d5edd2eea565042c8b58f765955c7515d28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/33c4744ece3be6d06ff2d73c3a64198bc3c2157b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/39c32f9575e1e40d764095afc3b17e1bf8a1ac95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/91023bf3a715f73a8b8c885507f03efb1235e632.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/b87a6ab649d8eab76e7d97ec4d067b1fa92c551f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647758.97/warc/d3ff47697cc3eb4f626f5ace158c71b90dd3d7d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/1af01c820dcb1781d840e7a0847a725d72add09f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/25685a88f70369ff3cffc6e5557a263e9b721c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/2bc71af71ca16b7b0c52db6fd5c09268187fb29d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/b1815fe535450d6be0996f7e4c22504e158aeb06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/c476b7ff68f07bacf43ccd4a3d0c32c2d71f8f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/edfdd5bca3dc7c17a171b93fc929a28acb651cda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/f60c810d7fcfa3d1350dd1a11954a42a9c4323ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647768.45/warc/fd8be378d3aa36677fe752aa8863a7a47d25c46d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/322cb666ff0cf0e34d1f0ea9f22b7de74bb6c6a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/4a4eae43f3f98e85a1a75cc78fdc1bbaa862bffd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/6e1ca71c099624fa822a6440c041ad35d5deb5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/774cad17893c159f65945b89e4cb84039e133911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/7c0519ab37a03f8cfc413d7ad824f737106ab44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/7de11b370d96f9173aceb569f7f75e866df7ecba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/7f081343175f953b4c05c4b7168c09d49a51985b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647777.59/warc/e161483dec99ae77772033a546e4a2ac6fa84f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/001b76a95b1e4e6e84161a434f91624d7f5736c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/09fe3ff70c645b3e7f38adb6b9788360b3e1fb7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/212e0016e6a5e445c53c07283e0596d2e254159b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/5339a09a9e222e7011681a24bd94b199536c1534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/606aa44ff8e6b264e2d00c798a8a3f00e0562048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/73251dce0535a7ab0e2230e8068c342c28f62771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/a5c5e4e57392d22b8df8d23262e742c00b20c31d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647782.95/warc/cac14e2a0a06565e200756be3a99827610945381.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/18df446cfe373dc92787218ccf5fe57efa9ab539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/2a5c3bbbf5b3bd955db62533b87f404e3fa4ea3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/4d2bb8832634546231065c529e85bd634239a92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/5e42dc5059c08b3216f3e18c5baf2c044ac2530b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/c5845eed0cc06e95de70465e87cc92419dc9365a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/cb5b97e8d09b60b37a2e2036155cf3f0de3c8df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/e309e8299f0127c2add2484737fe36ed7c7fdb61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647838.64/warc/f877a4ddc2d8e9a5f6fa9bba4c0acaabc6fd4991.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/1bd3865c88f03dbb3012fd9e03eada6dd5cf3307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/729f1eb0c2a6a216c6262ad60cf096142003c729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/7e4b4cbd89359f0d8ff82bbacd29cd1bb7477b31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/7f8e924eb03c00ea184474862d44e847d2b708b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/8198e9481aede76e2abcae407c1b345840a85bb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/9d20c3a532f59f212790d895e5495c0134331c73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/dd1213d6c12f7bde210a5a51c9378ce3e0c0027a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647883.57/warc/ec687051f2461e99fb71e02dfcc5c308ca467f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/105861fbae862e8526759d8d744a478c5d88e916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/19dae3505646fb1d5fee2f53cb79614e700e8646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/59cb8cf44c40c9a389e0f8ae0cd27697960fe8ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/6bb0bc4f522036e59880027b8f5e68384075e9c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/8de2b181c72fea87f6c79dc0e2c29c0ecfc49519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/b48130f68d1355b81e8be562305010e213506cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/c016c72066cf611f5e846cff980e4d06a0335fd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647885.78/warc/f89434e4dca76388ea977bcc69147c69fb5fb5b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/040aa8bd541514eae5bf3f663e0a93d7af858554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/3f147b8f1fa6a09b93b99e7e0fa2d411f2ee05fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/58ccbee75b01624773c12e83b8f41732d4324bbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/61335b7d53451d208da93d51cd88bb7f33c6436d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/ab4ab0c098982c77b20d8956e9f1794b0e7ce61a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/b2df8c7d6af8aa60c8bf2c860ae40d2386a21ea7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/b5c29041d6c7ff1b641a19cd12e98b4b744afacf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647892.89/warc/c0fe1933e1742d4ccd175e97e5524dcf0b738338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/125cd6d7d6b8ede7c438fb62717d53af712bf465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/2fa2bc16ca16c62e4b736c00638c2e7cb394b8a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/3e45fbc139756a55adc30e312f271a26fd9a8043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/46aabce28b5b7934a0af19ed66bd879592a8d8f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/6f15c87de3e2d6b07a1470fc5af086eaea3a6411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/8f07a782f65188b379d5eadb67b98053ef1f9fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/b493061b0da02499461050a60caa89815cf44292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257647901.79/warc/ee886fb31ec13cf4dfb9870de650e252d504fe7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/3ec61b85b8f5e5388d4d0c20e65aea58576865d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/4c7a7cbb5d0248a2876dff15fe7de2a887137aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/5f8283d257fbf7efc796686e3f81be2923d6993a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/68493af833f1354da22e7f3f09df90cf844f1ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/7b1cbabe1c75e6f8af5ba3487d9e62db9fa6cb55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/8f240ca6cb2f9de6e00523fa5d07598861f96c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/d0f9a4691ed3507bebe1e76a4481b27577dce074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648000.93/warc/f9234dc45d6a405704c2854f19ca8edf3f0a60ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/252856ffcc9d9b6a5617a2799ee886028cf94a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/381e7976d3741b573df2f5477c72258fa7f89ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/4fdac753d2344088f4cce4fddd4d5300cdf78246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/694de5c7c64703dade99c8439ab7948268937539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/6fb39309cfd8031e1ddaf188312efcc3aff7177a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/aa7e2304545055fa5db87a4cd1b661e34db8b782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/cec557629f2d664f70aca611b736e927743974bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648003.58/warc/fe184e18b8fa5aa81ad90b95199ff955dafac383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/02d5710a27712d28cd5125d41f2fb60e74d166c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/1703a14014e887356535ffb4b80ccaa6fa6df47c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/25d076b8c4f9b8857cabe1be89357351828356a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/2fd8693872a415f8468752db097a856f3c072ed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/337b40dfc59adc1b7fb40bf84cf17d5448ae0be1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/8640664d2f05847de9949b401dca79e9d6f9e3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/d576c75119fffd399ad8bbba3dfefa7bc695c34d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648103.60/warc/e880178d75fc0db12edec92f572463b1e58e970d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/0646d50e4fd0f635011a7d2afa143dbcab9b3f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/2b36a1ce5240987079bc8bbb1a48c724e39dd39f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/2e067cc2f809f1c70ccd1d301c7087ecb70cc56c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/4ecb07d172eebda5defb065b2d062e2525ff04c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/a98db8e10aee638189db40494143f011b2c63c80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/a9a7aef87ad7a7870b65fb58acbf4ee383646ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/bf6bb27af9c510669293621ac185c94e0a01720d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648113.87/warc/fa72dc4b6109814b4bc66e327355496be079b4f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/19b7dd0ba40025d0e2b0d84e5cb338273acb7ec4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/236f6a41ef9b664f81d2c9c2d308cbab6df304b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/272db282876abda8e69f8ca80de87330fd253d0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/40d4e416f1795cee9ea0501aa351910dbf8de013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/76e42bba800fc700707cb8ee41b6bdf905bf6e2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/a8862374a95c1dc9dab95fb072568ddc7ee7521a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/c781618d4c6633422633ad0b46ed5e5aeab17e31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648177.88/warc/f3592fc0c6a7c27061c1e43d5baa48ea1ec72c98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/800b1ee6406d5888d37abeb96b2d966151449829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/85365c74ca331d2c8daeca46a963b1ce7814a194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/8750a4d7dc8d4b836102f6c0d2c92ceb9e77fd70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/a523e1161799c297e15317d9198be3e9396f734f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/afbdc4b58f4db89deb3e55aa3397188987d26c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/d9a76d515bbd22f3d130d0cf358bc790b1e7a483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/e4b18beb30b7842a01c5aafa86d37d7b84b8dac1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648178.42/warc/f0f2c1ec211f673ba8c33fe2f39947dcceb6f934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/38160a5ad170384e538ea018048b8f700f1a4f75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/604f154cd873d209d7211344c1d0854556d8647d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/665af0c9d0732421965658255f1feb98efac7156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/73885f03697dcf95fdfe703a8909587ce76c13b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/a3adff8990fbd1e7bf25ed5e63e8f9939fbb0f42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/c6d899188e8fcf4a2eb4cfd1f9bd1424b18549e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/e34dde8ae88e2dea9bb980d74945a729d1b39793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648198.55/warc/e6c0955258254b93cc79a37165e47e86f234d3a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/033b2db086bdc6c4a6bb48c98e18ca5fdece30bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/12d6e7e35c80b2fc5c105035f9a07cc5fd5f2bd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/16d79b09fc33e29dca47f8a0984cab703df476e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/75945fca454f35685dfffa5518c3df451099aacb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/b05d7453600d03f29a02e0b51506fae821aa5bb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/b683c7240496252f429b3cf53aa93ba2948a26c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/e1c1af57bdd9875bf3a90da30e51efea19c50323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648205.76/warc/ed9e90c0680efbd71f126f7b1124802bd76586d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/47ec2a6aab76c4749db97c179baa09a4a179b78d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/511a1477e522e83868a72092a12a3a36635cc664.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/7d73a6201f8cbac53ee652b4fffa70c28c134430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/8b4b692f358eba32e4baee1e7cacea2cb79914fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/aa34a797b0b437acd1fd22bd4417bbd264b1626e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/c8c26069616beb697aab540ba0e247636f449fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/c9f6d25e447fc9db09fa3e404f0db8186402f916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648207.96/warc/fff8aa54a18ac3139de1c6cbaeaeb569a7daea70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/1c96a555d666a57fb12d9a84eec4750b33b9d98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/35839f2ac213a2600a6a7865828dee79fe27c755.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/3f5455ec1375c4feafcbe39e358fb07c2580fca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/439bf6bd9d719947edc730748cfe181679f8facf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/51e6d66eb2f499b7087b76828d1c946a19a24d77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/9d0010081fb05aa7d08676ff86b5ac06227c79b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/b95c218d25dc0d820cde8ec4764f36718472cf03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648226.72/warc/d998b6585607fc74a77ad66d4ccb86ef2c259420.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/0fc724ca0b6a144f1a3abb1c92851c9f1741b64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/2e25f7bda05de897699fd1baaead85c0d9223949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/94de6ce15127920fe658c45ba5f55338804f5626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/c71e147b01416738f20611a17337395c1edad88e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/cc41fb4c90c5ab2faa128744ce0db6d9b4836a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/d94c9845e1c93589700bc9eaada85199e3193d61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/ea32bc6254de1690878b9f21075911d28ba31ccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648313.85/warc/f935c49c3c57dba2e6c920772e6a34c35ba07f48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/2fa0f34c0f609ab263e4b9582eb36f0fa43f03a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/44bb1baa591b235585ac69ce7f1458eeb56df8b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/453416639344c01600cf8c78a2e161bc803db77d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/5a5b8c2def939966c7dbe89d3a99b327998d1952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/732868923d5a085cf3ed1fba228d040e64c9352f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/753cffb2f7ef2bff01a443af6663101d7001d24d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/83671740cc2c42f4f00a525c601a83ce1306f1b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648404.94/warc/95c2308547a0c93b8efe32ca5a2693ef94670e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/4d6c8e963d89a23b0c6e22615180318f434f997e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/66aa04272e4573d2a2054e07c01e52bafb57e984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/8447ae4ad906d491e9c696180f645f147212eff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/8d8409f84c9173152134ddf77fb67fa327b50632.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/a19bd0c339c3feb15e6c44309ada004c239612f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/a663ba416e711ed5e51820d7ccdcefafb33eaa4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/c6b1f6bc17df647717bfd67e42bfc9d87f927c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648431.63/warc/e32ad125c7a8d855251ab8e6d6a5e1393d562186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/057450a0be221302fdc9e624ea55f115b23fdc81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/0aa5ef578cea8b2a35f4371244c399dafcb4f1da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/0da12815e944e69cec37024eb944255fd8642d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/39b9f48741003d9c806457a51196b743306ccc72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/bd0bee19fa365cbc549fa25a191f85649185462b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/bfba605b17551b19066fa3c3a7438f43b723281b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/e9679ca9817af25d99d805189dd215e3a387e732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257648594.80/warc/f962384610af8f9259cd950fc8fcb95615fad2dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/002762bff232dfe89546007cf9124492222a11e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/57fa6d071717bf37545cf00a82e59bdd0a1ff479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/79fdc7c964d18860d0aeef7648aa17be0f955d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/c9c2a60fb8e8cb5b92ed207c3b8c20634fb747fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/cdec2e023b35bb7743331ee443636ea9e96f22c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/d0697818167c7b79586258df761f77940eaac164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/d13444ebf5b4993fb224802acf76b0f862bec315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649095.35/warc/f669b34b1bc34f7071f400f5c6ad9794aeb44d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/1724eddc4bf98e8088ea79a2b3b2fb590b3599be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/25bbd3c959b24f13418730824912b44a6124118d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/4cb46231eb03f4fb3e3af4fab3a461eb2ba75eb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/5780a101679d04c7090891a75ddff148b3b188ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/8b6dc70953343cfd01b72cc851f46962c7e6a943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/c47a5c86b614ce8e001b3812574578711097be77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/c7e174814faac1fac2dd664399f36d6d6d1f982e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649508.48/warc/e9313b3ea14c4ff20f7dc68ea2b4c803ec6bb56f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/04dbda81ff108130c65389013943e6fb3aae3e6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/0822ced722f220d546fe60ff9336ae342886de07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/28f1ea1be85e3fcdaf097230dbd4e88faaab3109.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/2f11f2bef69693f9ba18e7b9ad403dee1ce8c217.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/50c3d2fc586d23737090ae96cbf6457b6dc71ac8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/6763984695a2cfa17e38786844135d6614014f96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/e9274e9b4c2916d50e78d102cce042d4afcce95a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649627.3/warc/ee2a748a564d59cfe11cb88a6703c69bc85cd68d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/1ed8907b34b9f93c22799170bb79ac1b2171d7c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/30526380e469c7456a5e9a5392c561b701de1b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/5855271a296e26f227e4c4004144b855d35994f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/7dd274314d9e72475283cbb6c1923bd06548e903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/9ba36da69f73a416f2a64e1af190ad7fcbac8ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/bad316605dfecca29cde8a0e3df3f16130b3231a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/d945d36d51302db458cbaf91273da5ff237c528c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649683.30/warc/e297e9bcb09a062fbdeca9e52224418548375fa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/1b9d18b09d3fa841fc3febe6d201439e733ad5de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/2f99f4525b2a8ebf90060b099e8307ee714bd169.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/6e99c145b7042b6bbe09a0f47c33a6b35d088799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/781495da4ba3e34e58737177c0d70c193c86a754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/988689194789e2106fa98b5cb71f440cc87f1a09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/afc060012c59f86268ac0e8a7adab2c0897ee6c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/b847fd1c4f88952e3739919d282598b9a9e89bff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649931.17/warc/e61b6dffb46fab8abb90fd63947d962060bfd489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/1b35817726f872364ed12a8fd47cbdad3b7ca40d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/3d9f2943486be52224e37a2bc64bba1317052266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/4b0901c786b9cf22aae7e5f99adf83858a02457d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/8ea9b59e33bf4e523baf99baf537c5ddd7b08f93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/9cf84085a5053e8fef1f603e3edb5c90d06f0dbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/c5c28d7325879839242198d435b2cadc55468953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/d89e546c769fb067d3c956c815061ec0715b66ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257649961.11/warc/ff67923dbc307761972ab0516097388d167cbbb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/0f8d96e9caa5aa3949fc5bd83042d8d4f2fd05c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/1dc003521ee9116ea0046c8a23936513e0265700.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/20f00f09726d85db6665349a80a13078e5b52741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/352f2d14b935449a9372d77ca19f828e32a0e8d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/69ea1f65ee10f7e3205e987da6c1b674b4f3b5fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/b093e70b2b3360c7333d78ef44af33105467eef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/beffd976e65af7e693794199add6a949970f1bde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650188.31/warc/efab9cfc2602e5efb8132ff7bee65af20bbfd7bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/1330fbd97e7e8cb9f267de1385e0e14567d2c229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/17910f18cdd67960ca23c74b0fb98ff90809961d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/5538736a3acf1886c41de52f35aea74334b77bff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/6199e469ab80e680cacb06deafbb339e2c0a4ae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/6e20fc5370b1ab5886e2541a26fa13b787afdebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/c11f8bdff01b954c854182c531174ce779c7946c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/c3e97a6bbd4f23ff9e1e314aae62a120ef56dd4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650262.65/warc/e149b0ced407035bb613e4d8b150b52bc15e247f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/06ede12bb09b5876520ddd57195de420199a7a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/10d8ba2900240f617f5fe8ac62265527c336a101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/20cdcaa4f8b3493c54efb716dc72fbcc9bb8d455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/975ab11b74ea054cd64ba9bf676d3f3ece49342c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/a15c1bc4393b2f389256358df1e26b54fc4ea108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/a4a9ed0a8bbe81e8b031adb5327a49813f55e3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/acfdd0a5cf20e8d9ade2a094dfbb46aa05445d48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650685.77/warc/edc69acde4062cadbb6f06e16650d80be4a9c41e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/0bc830b3680e12aef2c8bd7618c25c8c0312f16b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/1eb5c207d13ff1fa1b4be03a235f46eb920f612a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/71656d8c230bde1251e9b338443e263be2f67450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/c41d513a4730a4022c121f8c06f4083e3ac32c24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/c957fd92c9f87e93708816c3d51f6213d3c3b216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/cf187e6e7ff2260ba224516fcd959b25ea4574fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/da8ec976ed633f66f329359952f1f5fcf6ede48b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650730.61/warc/f7aee0f8c582f37469f6b7605ffd896cec158ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/3b810563a77cf7f54d9069ae70374491da902111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/4c67fbff2265167ec58e461d0b23976cab81692d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/799ce5205817747f8097a14cf40ae1353eda98a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/89d3092541a0ef91e194efa9963c873a4568b444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/8f5b24be3f1c6e2bcb86b41ffaccddeb20639cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/cf908716c28da45b2e68bbafccf7a5d44df8930f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/cfb4af0badedc822c9f9b2bf7b4033bb78fe7662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650764.71/warc/edffb084f2700c4b10163261168f4d6dd035ab01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/2f8b88e008e6ad67bdbc482b47efe2f6f0a42303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/8b0a69f73dd031084ae4b184e01704e054eeaa17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/9315720898149816d00fe44dc4c98ec70af73586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/a578c3c93bfa1681e0ae6d11fb3a5a1c4e544fab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/a750f5648773bd833711035241ae188768e0b428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/a85b8280e91465a1ef362af24c1f315b6902a4bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/bc75990ccbef187f1cede944b50599203e7887fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257650993.91/warc/e5fceef06c86ff87b2bc19d04d77fa16cc655e0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/41b10e098839af13b98c62fed552a22f0c137118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/46bed51a25abf6baa030c1713db53aeabb165f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/4ce081256903fc88b7ab650817f4c1f656648376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/5337aeaf4b064043f3681ef43d44fd63f1ef7255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/798fcee13bf26b101f3c5342e5a172479a3ec79b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/b8993a5e44531ed760112ac36555efb70b3dc1be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/c615ea81556346fc3c04e66b53c3504397e0c91c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651007.67/warc/d528baefc83d5192dfefccecd029a7e29e6e4fb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/1103d2a3c2de39e513e9d0ac3f46c3663edbc709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/40811d8f2ef174f17acd3490394d92b775990cc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/44a8315ddd7e30d3ae7d0f2091fe3ccfc2b09ff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/46afb6cab6cc8737d9bdb36c3d994abec4937544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/4e1fa00ad2c00ed59eebd9a3ac7ffa9e2505bd5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/63bb62b75a9176ac2a2836c3e2dedbc02f230d29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/b9e8284952ca40f3d11fdfebe6eb2d0e967128cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651465.90/warc/c683ac20944c07a8facb8d80ae103cb48a2a2d0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/0e8841ece651178964fd43ed17813385e923ea9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/2983d8d02879070237ecc8e48d8738e1e4fc6b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/3a64405cfb622bae312ba458930e365f173bde96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/3f173801a5bd385d4a97cdd401c211e90f50a5b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/a311bbe8530da4c1bb94e53fe687c34d2d49af2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/c3856647784365e26e14d2df055a7279465db8b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/d0c2c1f5685f3fad11dc0bb60a2fca5bf398bfa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651481.98/warc/d12ff1ff5df0134554845c0cf7ea8fb03c9745ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/110ba8927773d346cede80bd066e55aed26a8229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/399843fd74c96c22a76225897922b94da36ca25c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/5cbf251accd51e094baa87b45f1887e0c9ceb307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/748265c72d486824343d4ba38069c78e676ec58a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/7dabe53b18fc659a89e41b1d24b8c09209de786b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/8699bbae76ba49a2de6f79e79b5e4d86ad07c43e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/c1c7afa620422215066cb55039c7143935245e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651780.99/warc/e06820538b554b5a29a2bb16196314afa77acf92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/19a7902e909aa8cb16b8c409ace41c685cf2b424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/62898ffc288e3c038a9da5a083defb4cd5d52c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/7c28c1e45305cadd387d4360e8c832104123ba08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/903cf5368bb24385af2717ec7c0488ec94d5a81b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/a56c3a59dad4c5e4a8ba4194a169681395dc0135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/ba8316bad81bc377236c55e572c901d0efbd40c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/c68ce2284b3ea4ce255988cdc8e6342973f601ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/1521257651820.82/warc/f39a99f93b7f202052a757e9e7bcf793dde3e249.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-13 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2018-13/mix/input" + output: "/tmp/cccc-CC-MAIN-2018-13/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2018-22.yaml b/configs/cccc/mixer/CC-MAIN-2018-22.yaml new file mode 100644 index 00000000..f81cf874 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2018-22.yaml @@ -0,0 +1,754 @@ + +streams: + - name: cccc-CC-MAIN-2018-22 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/04fe445ee4a3c66d325b7677a4bcd21d60b6d4a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/0775271c96a91f6010523fc9d30886653b88bd85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/08f86a1d660eda9be244d5c2aac707751a91349b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/0cc7a6668f632308a51a8aea2a766ea9e7bbdeff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/0f1c5ba95f3d26b701482becc0f78f77760cbce0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/131cfefd9212fa92660ec84a99e97f2a5bd9752e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/1553d2465298507694e710981ec4eb661c07c1bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/5249661b8560d6367f7a7c0960a2514df6e8ca56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/57d7bc7fda7c4ed4ce74421b54b493e8bf4c0a01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/8f57f67e90d6fb0c17427e2510d18922f05c5438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/9ceed74fdfd42a19e5713a35081a6514d8e063d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863277.18/warc/b56bf693fe4d422d67bab1a39890caa877c8f648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863410.22/warc/16b55b0d420a034927d58e4e47d2bc16ddd176d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863410.22/warc/a3c4230dd583768fffdc7739c74a8f8d6618fd61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863410.22/warc/ad801ace626fd2973ac530d83169d5b83f8ea434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863410.22/warc/dbdc2d527b299a4fe6005c738f09d72d953b2f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863410.22/warc/e830db61101fce9aa0d42ca58cd4bc4f66f6c21f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/4810acf2c6f77d4da73038ab7e0925c50a703d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/60800f0a2297d18a98acfe0f19a0b184355a9eec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/ba4f939a8eb9d00113bcaffba7d3280fe53ab871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/be4aaf8583800590d50f637f805c6bb76f8dd095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/f81dc6ab3d0f810e4f5d8a71416c028a8009ff25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863570.21/warc/ff174c576cb6c7286e6d6d029ba29ba725d6cd75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863626.14/warc/15f0d99be6c170d07807aae86ed320209638c296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863626.14/warc/2c4da7dae2d1ff680c232c12dcb528635abf8ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863626.14/warc/5f0133992adb3402c5df37e63e2ab82b1230e2d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863626.14/warc/6405634958de08a497e8b6e36acdd5afca3ecdc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863626.14/warc/c7339073a95b480e93ded006b5d03afb189e7865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/26e709cd0b25380c0218edc0d47a6723aaaff2fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/5ea269e6cb74339a174b48ca46fa3b48ec4a2cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/5ea4d70d8676f5f15bb49e8e139aec5387ec9b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/74a1e2449a775622061d51f6aadfbf414de430bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/b77fb6654d0cebf98bc8e636c51ae1f4a5dd2fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863662.15/warc/c69694aab9c321eca039e5485d44f6a85e5ef908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/09d6be2ef26d4e5a253b978ddb9a49963fe4c36c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/2cb3286a85a6b08bce2bce610a44d1a795d9b790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/93d3efbcf2cabfcb8aa6ffdb5561a945d8b1ff37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/bbeaa539db900a54ed765d5c86035556eacf8966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/c60e6d2dff5939073389f5c941f305d5b0f49b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863684.0/warc/fe14dafe458dd12ac6b89d172a09bdd84aa01b82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863689.50/warc/1357788b6ca5b68e4b260a83547ef0f2fb68f7f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863689.50/warc/72c87ee0e2a53fc7775c989454e1bdfbcfd2738b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863689.50/warc/a9dc16479e74d2751edcf0f7af9b81f3dac3c741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863689.50/warc/b08352d4574d0cfcacd962ce98b9480d67472111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863689.50/warc/b47af452e802075dbff4763995d22004f6beeec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/0ebbf319ade7451158aa130a2928f6ee6aa87e79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/1ca4f53d31a90d63da5e85558dcd71b5204078da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/1f71f28ffbb91cc5c51319314296aae67646c52b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/61988cfb9cd1cc0e9fb59f7e1a0288a4bb8a93d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/6c8a94e817f455385e143c97f505d6e6a1a24ae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863811.3/warc/b39f80db12c9780ee6d8683a147d924453679231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863901.24/warc/26bedf5d7a624ad9059a753e296141628230f25a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863901.24/warc/325a41c95a0a495b5b650661ab568f9660b71158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863901.24/warc/94207756f48ad56a4061853cc8475ec73d556f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863901.24/warc/bc42899b4b3c3fe9ad0fdbf8945d2e23cbaaa552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863901.24/warc/cbc282399177d71e8a7558b8dd10e012830d4867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/0204a0452b3cc09b2bbe736c67332d3c4ed87a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/171b51b2c510bc7eed821ad436231d2d3acf060d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/44a0024fb2f6885a370aea7e78fef870a7190264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/97a1d3460bd49279e0c28114fbfa16cbde1a4568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/cf7c605d89119c1c2cf65a9d9ef9b3a60e257dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863923.6/warc/e8fb14dc380f2459d37df7a9d6c685e027e63a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/5e4825c35a5b8080bc628c198fcc88b8e17b63b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/7fba079b03b3afd5510d369496743ea877affe50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/9afd34fbc296a8e0f32e13c69f7f77de5b251696.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/a01e039a336cbbc3c03e7cd3e49d5118cf3e3b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/a873293a97748e01dd07e4b2bbb63c5935e218bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863949.27/warc/e7d7dbf2c6fdca595b204ff3aab3dc3f708cb0c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863967.46/warc/10e5876f2359dec01b6d6773596f53f67e190b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863967.46/warc/1becf5a08424832cb96146906345a26de0d4f6f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863967.46/warc/9fa4a3cff294346ed03d84b176b2ea4683c54b94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863967.46/warc/c0debd3e7692c5bc676a03c60bd2bce79526c108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863967.46/warc/fba2021ff33649950b7f1e315915b3e3f49e7e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/24ec183c81dad1824032c41ac4280a25f3255392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/30d7a09573b42f628887d56dfbd421c23147c310.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/3a4da7914b1a158904d18a61c7cc5253a12e14a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/47b65e75378eb4ed92cc8b03e7903a2824ef2aae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/62924bb1dbdbce03f96eebe5a9e76c9cc9c8ddda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794863972.16/warc/eabb886fff651585362d80f69f6ebbd8546ed613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864063.9/warc/00b3be1e1edf64dd7c16122d17bfa3b0addd204c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864063.9/warc/3a337ebddf3af8727d151a24be035de8ce2c49f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864063.9/warc/3b2d6413bc687ecc067cfb8c5eb18846f65f67dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864063.9/warc/956c13c8d152b8f8d8a07b758c47ddf999833850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864063.9/warc/b826d85202e14004e272482d15a86c22fd86cb01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/0fbc31f92ab92b9181bb0456efc9e65f6183f5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/3ab64422010e74689fde21082927ce6edc0f9483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/4319952b55f317d017692bcc418bd31cf25dae0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/5b06e8dc39290f3ecb4160c378095696a2f74a0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/6be329e058d4347271f0fb5c39fe6d3d9a3a3889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864186.38/warc/aa05c400d852273dac996f8b75ac64cbf5853fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/3d70606feb1e13f34dc6e4e0f29429ec4eabdcb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/8ac52d57ddd3909cfc391c95fbb1763bcaaf85df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/d115073dcc3be3b6312459470716afa85a7e5613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/d173241354c7d2f93e3c7558ec64aa7b6565aa0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/d9fdfa93af464ffafa236ae1deb11648e121f546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864405.39/warc/f5b065d344a2dd5b15d465b98ef59cd1f3cab39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864461.53/warc/30e4d4455d774cfeab80436598ba9cc0ae9369e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864461.53/warc/382610b3fb1da9767619156d1440fb8a35729a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864461.53/warc/4d7d116b14a14d47968c91d0f6cb0a19b094c13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864461.53/warc/752546181cf116500b89924c56e82643e1d0cf05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864461.53/warc/88327884034b2f368d789d0f2bbd1cc2d36c9116.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/1087b6547752786033b47f5908c8def965c4b91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/18217688cc8b3a67eaeab9a181ce42c50f2e9050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/6df2ecbc1007dfa65bffd17f3a90c11e033e5e9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/8062217d5f3043fdde28c7bd59598b9f0a3aadfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/9021eab22726818fbd94ae1fff5eb4ca4eb01e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864466.23/warc/c4450050b464c084058976834d532e656e37ba90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864544.25/warc/1f6cccd118557bf1c9b53b30121278556024a48a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864544.25/warc/39b0b1d83acaa5a297e18a1828b1f129dfceb77f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864544.25/warc/b4e0d2a4a550c3774dd2d24bafa7660dfe8600f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864544.25/warc/b7e551d610986a5a5deffb6a67175db66b721c99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864544.25/warc/ccd93dc5c2514f9f93df5904a8592836102366fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/0969005909a018b0aaa35dca5954fb2e95bb5356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/21ac169b8baba84fe676e954593bb86b274ad678.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/49ecce85875487104ee4d9fe62de273ff8306d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/52ce3bb25af930d6c63f734b587540f948d5e85c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/84348b969edda25a8ab05c7aa613ff6ec9cc2c2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864558.8/warc/8bdcbfcf84d3099c63ee83e2d49b3733932ca133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/2978a2e221175ed75887396c3667312f053bf40b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/3526c5a9828c0bab5d62cb96fd8242bbbbe7cfd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/5ffc8b2a4ecedbe7ac2259954d80ae5e6cda724e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/6523fd53998515b6a27facd75cd6d7c42e4b327f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/6a0d4f2616cb6745b56b0554aad93921804fe1d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864572.13/warc/ae4aa4b5dac21c1a92c118b746931e78696a2567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864622.33/warc/21c03441d68e81d2899e1361b9108c6cea85b42d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864622.33/warc/34f766e9ff1100fa100f2c9ae49d45685e725269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864622.33/warc/772be98565c5f45f834fee6f74932461b483e134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864622.33/warc/b942a41562c82fa630221fb1732b1f422adfa6c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864622.33/warc/e13a12f0aeb86163e26a731c806c1430854fb510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/051d258befa3c35729fa73b303d0a04fd3ea4da4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/2493e4dbaf9ec4cff766131d19b795f5476c1b28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/533b868c960f6b7d1d04b878cb689786541ac5e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/76343430989ba67534662fc2956b8df76ed9d2d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/8cdc196663e3802ca3bddcb1c6f8b5c50ddd1f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864624.7/warc/c7811821759979615609688383ca4219cbff6c33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864626.37/warc/0a76050e8bad3857e3cc9aef7792773aeda5c324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864626.37/warc/5f3a9bc487e22624ae597b541af4f72cdc66584d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864626.37/warc/7782f19aaa5395e09ee5e11a5268082e494bbafc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864626.37/warc/7a25e798dd02b354cfc1059462c127a2ca0a1f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864626.37/warc/ef40bd6d3be5553c0107c84a0b1de4b1d0cb1084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/06593ddd9d8c487520dcb23ad1a328d2b4ad4bc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/091b156e57d22fc78d2985351b61468a778ce2b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/0a20148d070accf899b973ec17b5e4e899a7c2fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/28b9e52e4c5408ea5717078b4e1b1efa42588dee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/9f4987326a0df8748db7d577a0584e2df73078fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864648.30/warc/bdae23b9115deaddce1385edea45b232320347ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/3cbab7b4b731e1849181c376279b42d2e0b08b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/92c6f2dac6edc852ebfb0f2bbceb8d1c1252cf58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/c004e5d26f2511821f375b7cded9ae513c8e61f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/c098b2d0edcdb8532675b8bd38b6a8a0ce524538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/dab2f74b86c4ae32dccf69e512009aeef2dec7e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864657.58/warc/e2fd6292f1a4ba4b28846bd7f35e5850942c9617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864725.4/warc/2289495dbcb78e467939fe18bfe2fafc7f871cab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864725.4/warc/50a958c769d7f7562892a95ba94241c74009d5bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864725.4/warc/62ba81a28f17a6c58a34ccdf968f3ac361d3767a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864725.4/warc/9b740d00a0c35fa36de4299dad78b547c19e12f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864725.4/warc/bd1c3494cdb052b442f3c59889c3f250cc59a33e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/13e6722f82142100e222040b167bde1dc70fa13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/583642be54f08e176a4a402b7a02366390a08a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/5a1d201905702d924a7f5a0d4ef7b3531e2e3ae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/9bf2166a5315b6ce818b9eba6685518c5588be5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/bab36d5af6f33ce5e364607d00bbefab67b648d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864790.28/warc/fc1317d17f8302160f84de7239d0c7244529cd2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864798.12/warc/0cd2f7b9e7a4bc371e2f8a76fb726e4a6de678a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864798.12/warc/47c853ffa6db03d29e0b3076a6b374b0f8f2851e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864798.12/warc/853383feb2a0658972a514a24eb3488ac3c9cea6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864798.12/warc/bad63ad7a635a76d983d12573568cbbb20f3a702.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864798.12/warc/f267d1f958d2d50bd4ff7e504e39bf093f7934e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/05d085f0b17673a78bb1ad709edd9b88cf14489f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/45478f559562a79f8dc24301cfaad6718ef02c62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/65e925c3fe64f353e262006d401395dd0c4075b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/764154d90ade14386477418bafaaaaab940056ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/c0cfa29b8f921c907e0839ee0982c1972e447f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864837.40/warc/e985bf6c5e44ad99db6084b7688568896843ed05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/88d33b486eea2998d1b3f4b8d63c12634d742516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/9de4e2487c66479b86f706e12af3c5de202d079f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/b9fcdda88658f86ac042dd2133cd49f05bb494c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/ca3cb6f0e0eccdd3529d4c4d3b778d7ad0f26b9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/e1f8ea496939d2a8aee74a146e84743de00e37ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864872.17/warc/fce07a9ba60195b8f42b4a98f8e3d6e6be93411c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864968.11/warc/03f6d515534c5a3979933186abb1e3a2b3b1898e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864968.11/warc/0ec0e4b497d7433ab601f5050257447b8aaf9553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864968.11/warc/131e5e52c874a7ef41cd1cc3393f9f48746c4d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864968.11/warc/151b43bc5e5d7219e3cbb13f55d497561e2a7115.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864968.11/warc/69f843c88a82929225d136e083342be992b9f8ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/1204a8dc8e4eb2557111c4ba67396f15bbb3bd8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/4882c841eae7fa4281bb587b482b34fa4dd74871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/76465a85817e58b5374165e17b0d72fb144c393e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/a8de22d11946acce19fedddd6ce6c2a22e7bcb10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/d9bb1ee9cd66c4abeda294cd5a883051322bbe16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794864999.62/warc/f167171098b084343bff1310a67cb84fa315e5f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865023.41/warc/2fffc24b3fb6b6886c0951162fb6dfedb4ea6185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865023.41/warc/9a11925be4a9f58a526626c3a81c78d690318619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865023.41/warc/a23433ad99600f6157c4598149f21c66f6b3d1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865023.41/warc/b287b2a8e4c39150548a272af69a1295607236ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865023.41/warc/d18b7a7597c1b53c09e04ae2ebce0f8ff32c82ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/600b9540bd1701a7e7e6a8d40bf203312046f6f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/96672d01fddc3767da9bb986e115ecfad4581cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/9abd58254271f13673bfb53b48b436e922f2a1b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/b248a86538d7c280bfa786902548e1eb71671209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/b9e723c65b634823ec71a1511653b7404f051be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865411.56/warc/cfbc4464bb685f149b56bde7cbb85cc514aa5cc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/0132ff911012681103f155bf5752337fac8069ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/540e36cf0923e40203291637a1aa77b0736615ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/7051dc5bb791966ce771eeeb9b0a7ff475760452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/748a77281693c0b2c108c34fd7ce6acf64d04f79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/d45bf4301e5aa3ad04ad5241a676b2c74c246834.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865450.42/warc/ec54ff9f0f7b6445ed103d2efe04cb83ad55842a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865456.57/warc/0211a463fbc0f2c3ae8e5101789d2a677fccd0c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865456.57/warc/30439e4128da4fa74283d108f2aec44d42ee6493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865456.57/warc/974fb3e980ca0fc42804c4f7f9f278fa97b3ae73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865456.57/warc/c73b9b193e53489ef4794d4d76d7fe8595bc6e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865456.57/warc/db687a3123df643213c64fddb7c4aacec765d135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/27315a247b3b76a32a8dd58ef7a91a65aef1930d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/3d67d3229bf26e9c4cadeeb4ccdb344c8ccac925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/774df7d3e00bdad86552f3fbc9b310c242c885b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/7c7a3770bfa82d00c47df2e010228caa5728e47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/93002ccbf16565d01a842d736758b5deb48aaea7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865468.19/warc/9bbdccd1a9e0aeb1a779b52cc4a2ae94e1751fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865595.47/warc/13ffd914088c9f3160a87156010040138db9d5c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865595.47/warc/2123e10d5f9a48ca960cdf2c25a52e793ef64102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865595.47/warc/4db5069f072ed575ba02fb1fc6331b8a34a5ba3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865595.47/warc/64358bc906cab4783993a20969a3ad8fed85cf13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865595.47/warc/fdd55bf8e8348c081eb972cc9a36e62562c8942d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/227dbeb3dc1c40c1f951a9d1c37a81ec3d205031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/c5a03ac1eb0deaf70108e229070d978394dfd34e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/d8930b01b3f6e2eaf81c0afeadd20eaaf210f007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/db37c16f1eccc08fbaa272af91c509c8039d886c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/dce1261f630abf026114961b6c949afbe19d0cc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865651.2/warc/df042d412cd65692dd0ef68ac2e29050183109d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/08458bd18482906758b25ebc81418d22f3d82b14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/2cca4a57fa10195a8c30fa5b70f7a2b8499eadfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/42a8527f3d7ca5b5aa2a3b02601f115d9843822a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/793c1887e33962778c2e2875c45a2bf6d0775b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/a84ad582b2ab5e0f2ca66da81454feb4fc1861c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865679.51/warc/b11d644dd398a2c7219f3d9d568204e6936fd832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865691.44/warc/3a2b7b8f2238372d051779cfd43cef535f6041eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865691.44/warc/699ee241037c95dda22b3c6d84810a85530ba033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865691.44/warc/810093ae01274094e1719fd1f1eb288b394e7bf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865691.44/warc/8b55ed7d6028f266f49afae5829045a080799a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865691.44/warc/ad079e02609e051c69b18349811d103a32f9959d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/1a424d240cf4bc9febe1395b6340cce6d1390024.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/2e8022f70d0f62cc20133094ceca787588a488a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/68daa99e1671ba32397d942c476763694ee0b817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/769233b948adb8c321a6240ba52b16a678582d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/c06f357d4d47866c6ef124500ebad95e9b44c176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865702.43/warc/f627201c391160a65a0b2212ca39fcfc7c599d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865809.59/warc/2dc9c0ef4ffa4d7ca76385cac2abbd9bb10ef8cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865809.59/warc/4c32d05b299e43a910f43e293ae28bfc2bb71eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865809.59/warc/6c771c48b1214f2041cb2e39cb140f8dbcf83d00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865809.59/warc/d07f70ace07a55c69967e59386d03d55d47d45a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865809.59/warc/ec2c3831aa7d2219288dc3b76b5e872e40532ad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/1474ea9f0c4535e28a567346cfd2370de262b0f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/2c032b16242c21fb42cd78573c935367744f8ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/50bdb0f1e3922562c07b4ed943bca6b929e7e0e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/c0942ef58576f0f89540c2067a4e71cea4c8915d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/caf2566e5f4a01ff3027a88160a3bb6633f4ec54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865830.35/warc/d3bfc7af64939e82eb0d888b6562c1e4fe25a550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/3fc2d1b95589576f0b315fd3d4272be1d46c1515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/4108a0f9cd5c1d19db3d7375445e8bdfb1cfae2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/500238763e3e70f508f3dbc65b491cbbb7036068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/784e14efb4e7227244e7a10b8e94073ed100fd03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/789e20da50d31ac0b4628042a9265c51025bad03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865863.76/warc/ccd937c39cbcb52f51f436f68722f0e13d680986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865884.49/warc/008351d2e8720c815fa66d2b2f0a08e149cf6992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865884.49/warc/1fb36052aedb69f2f925a7a064e35b5784cd2e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865884.49/warc/4e9d4aa58275c0a9cd14fb58a3492963e49ee370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865884.49/warc/7296a4f21477f9a0ad809438550a79713fb5e1d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865884.49/warc/f20aee20ce7dc282e8a09686744596615434e3e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/24179d14cd04005172131ea666db3791287d7c76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/3fd343de19d4e3d227271eae357034fcaccf2e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/843fa506898034e8f7859cce92c49fab1888dc77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/97dc6a66764f463d5cd3d7e5401b47e095479f78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/b82145cca3d08bda2f60562a929a5f27582f4b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865913.52/warc/b8db1325ea3b00fd4ad82d59f797ede6b297aa1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865928.45/warc/6be6116672446a7f847c0e702f56339711d8427a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865928.45/warc/79206a13f9f08d875c8ba407605afa4b00907050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865928.45/warc/88bd97a0a72c617bf6bfb17e41992cd23ecb7ffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865928.45/warc/d6b3c17332aec24884bed7a91fbd2e5429fd1c83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794865928.45/warc/e4474524c66144a87a1acf361c9b73589ab205ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/03d87d04fe601e23e51930219d6def764b44c1bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/232b13125feea15dff411a5fe916eb5c4b100ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/64d1a82fa47787789c33ae838c8ff6edee413bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/854ba702dc1b53d9ded10c7530738b6d3560e0bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/bb7690c7ef7b5f7d50ad34960b5a726c85a23883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866107.79/warc/e65357c2e275c9bc6f235938705c5f8bd8763b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/152da02d34e525adb9aa3c2000422dd3499486e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/18d0f407a229ed02a4599874acee35e0935fb4f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/4595d40bf9b0f94b76dd3a8aec3f1fa222a4a93b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/705210e33a211e21503fbc64aee4cdee6a7edab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/7a848542ca891d91a8276f350d7b9cd17848f599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866201.72/warc/cdb201d81b63a4ead5fc97b25f2fa176f3e25e47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866276.61/warc/32393c7e50949e4bf4b9319077f9186b4b4201a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866276.61/warc/71232e4f26b1fb877e190f7684aed0d2f575f1ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866276.61/warc/9aa529f7db7c6f3d274e9c932670f67a5487278c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866276.61/warc/b9d4f84bdb518fb556be8bc9a3e109aa7e290af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866276.61/warc/fd8f1676415bb49145e70cda05d00408ceec9dc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/0f5f0a18ebd2489188cbfb631445fee41fd1be3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/860205576a7ad8c308fe5a853125c417318fdec8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/9acf9e90964d7232917e178ed472ae86121ae8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/d977991ecad75a720f9767102b48ed32b6d34794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/f9c1a4b285f0b02e740488dcd4f4f913d91f6136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866326.60/warc/fd11bce1d72cac715f03bd358675cf09327dde0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866511.32/warc/11778f8364b03f536689c1c9e805ba3e5448e06b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866511.32/warc/27c9932b30fab9efcfd562b978b49d8da0fb1da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866511.32/warc/4c8c259a48c8fc97e744c42915235784a811da52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866511.32/warc/b0d6285b3352af283ebae6137f7474cc13a974b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866511.32/warc/d1defb7daa2df613500039ca5c9dbb1c54fd1a6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/489cb7f8d65d90db89b68350285687c574cdfeab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/4b24a5feaa1b7930e46992378b4104e513595963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/61e72813823c8e239ad8374a0f56319d2d664edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/9c6972dbbecb7584a3383a60737ef92cab726a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/b0e7bac9e1ae54ab5d5345eacc247705eaa4110b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866733.77/warc/e8e94b4b1de1f4ea118b5d506ce3cbe565adb57b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/0fe1b68c863197200d5ca153505f71426e1558d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/1d8a5a380029ad6899e211a44278d20d1a098445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/4aca7db7fb9a3e727a8303352223b47890284eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/5bdb0281bcf755c8c2f9af24ac659a1f13a1a775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/89bdd89fbf6ed89c57780bbfeb0b7ee488b7ce2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866772.91/warc/e9c7ee5f51ffb529384b3c622fc769b09364ee37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866870.92/warc/174d6d8fa883076ab4cd80b2f026475f6e334703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866870.92/warc/40c2aa60b119eb50e0b62f511a7aa0b7e7f51b87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866870.92/warc/4cf8a3799993d68ee6b92ad2cf46b9c367b194f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866870.92/warc/b2146ec9ead41534dc6b964a69f28fcc1de0e3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866870.92/warc/da0f9f189aed5e3c344149a83773157003ad5d6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/231f5113827bb2b2b86508fb9548873e87cd7d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/8b49d83eb0d6838be751b34e3f695acdb83393ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/954d4360cde7097ba7b08be8df36b77e0ce0dc56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/a67ab7210a67f874327768efd68d74386839f675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/bcc772b9604fa2df2d1c6421d4ffb36ce79f9505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866894.26/warc/f72e0587ce7fc9bde2a4b2515c621e3f29cc4a1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866917.70/warc/34cbfee88a1bf115e4f7ae4203f390f736069d71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866917.70/warc/a91d64c44d513227d4a0026cb89208b6ee3bf187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866917.70/warc/ae52cc3fd1241dd953e6e4a2a21205cc2c701912.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866917.70/warc/cb2c7188b94b6e82adfcbeabec6ccbe9496d079d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866917.70/warc/f6b44d8ab189be9270403e4f675cc123eb4582b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/1ec2cb48d7a760832e3b76457863df5c4d27afc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/264c2521447d217037925ecf17e6c445f46a9171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/470ea63a31eba8bce1efc3d7db11d231bd53412a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/cc39886f2f1dc645974cecaf49c2657712b0be62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/d9e85d41bc56bc2e75587cd9d9026ce501de7b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794866938.68/warc/f41eebfc6e735529df3461a6a75e0c15135e2f7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/84652109b35be9e3bbb7f639da1e24a6d6f0ab0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/9350657936291ddb44a2f1f92845665c4b0d05c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/994258dc1149342f821d7d6a0e053643a1c28d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/a1a83bda1c5e684cdaa68f84cd49c66fb7a42df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/edbe70a29ce07c8db0ed8830e57fb0f3ab5b5dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867041.69/warc/ff3deb4d303895407a3fc590704afd6697384811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867046.34/warc/1c41845bb22d38a2b4191d371ea3639b2dd40b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867046.34/warc/1dc818170759381f6ae469d0f28435c504d6c393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867046.34/warc/759a063a524c8510b6fdc3a44da4e274dfbc295a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867046.34/warc/d6916f2d38ff74295748b4a482f143e4f72b7ff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867046.34/warc/d91672e3ac2af1eb83c5cb9ea6a3478e4a9491a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/3236bf224581b73c2774b66ccea105371dadc2aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/4cc3e12faa56e927c4f026f9a1c1030818f9781f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/8a5be70f4572ec6269d7b9a20678035f91fab402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/8b2e41b864a48662534062dc8dfa66b054b6d772.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/9e10ed550ebac8408c40b39a586d5607bc4f2007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867055.20/warc/db7ede25d98c857c4ad45201326567a1ecb471d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867085.95/warc/0a0a50016bd3a43d1c39d5c3ca7749a8365009e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867085.95/warc/0f6d02c98c2fc9266e576b11709bde5d0cad20ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867085.95/warc/b5e2904a605b61b0a256e51247b949a96aecbb8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867085.95/warc/dd553d1f8b796c3b1648fb6a0f2b87ff7b212aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867085.95/warc/e5ba9dc4aba2b51e0761a300c4f6c7903d521623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/0a777a6f2692f3d271f482445a443c353664bb5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/15219f1f3c4f0b6a5c05555a9a8c1234657df94d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/18ebe2851ca829dab5583b795b65da63a786b892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/1c3f585ddfcbf4ecad789f0005af94891536e540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/812d1401a0f2858c68eb68553fe64402774b0c0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867092.48/warc/dd0e48901b02e469088e9e40eb8eaa38dd277050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/24521f7c7381ca3c0eed17ab5d277b361b95c7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/3e4ee66d15c565f7d349841eec4db88e0d5f8e32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/75aca4aa6664e0b496e5f45539259adbc9a85f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/9f54f69ea2cc80089e7e0a06835cc7c17f1913c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/c711e99511e2e06ff59e57fbe08a324423212f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867094.5/warc/c9efe24f617e2236d9ffe70df1c8c0a7b437574d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867140.87/warc/1351570d32c428bf2a887a1a3f51cd710c7c1680.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867140.87/warc/8058fcc6864ddf3d142ab830fc36e69152e564c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867140.87/warc/864a3fee8d83ef4d9f9e47e77d1d0d8b6c2f97e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867140.87/warc/e798121a9898e08d0a00ab5f38549893c55d1105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867140.87/warc/f16c7783e384c123863c8d0b6a83595b9eef404c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/0fa3babd0c9ac1c5238c872b83d64611522bee11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/348e89ab18dfc950777bc16f4b2570b66e339cb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/49ca1e278ffd09b46a6b0338e752390ea2e984dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/8dbca07080a5a541a30611be39eab898fc642c1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/9f5600780366ae9a0dd80aa83a9579d5e59f76f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867173.31/warc/f8a1ee67b9632104c28c322b66cfe9baf9af5e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867217.1/warc/6a210e3da771a6fe1e0d6137b0c87d08eb591d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867217.1/warc/9be4027ab49a7b29bfc7aa2e5daf33c1edc44b10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867217.1/warc/b4b45c802232ab4ce187c7c537514f5c1a529c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867217.1/warc/c51011bf7a2b9d7bcf0546492b91c5a2733681a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867217.1/warc/e05bbe58e07c605a789e512bf3de91466c8962c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/7969f534c04a6cb497333071b237c158b43a9191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/8dd4296f841771dcc871abaa702756c75a072be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/cfe859f70ad0f2dbb4b5965e737a00925fc402ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/d321e39d9caa5ea7352d1ad7416c53ec5f9eb821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/d5a6ced4cdcad9a90d6c4c16a23180281c255f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867220.74/warc/eee06829dfabd0b57e375d16d1717138f97db75c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/2ecc4cd2fc63ecda0151b7589c8e51c874fdc2f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/3c97a0e6f5b1dc17d367a2917fd9bb2f5ca8d14f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/3fca576141818f16e6898d74e301df3a9cf353b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/78a686aff16afe1d041ca371054d8103a0abf055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/7ca759629571d61ccb66994f163e90ac4bdea8f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867254.84/warc/da55f83919b2bfb7a014b60062a82150a54466c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867277.64/warc/33521da09ff8393fdc92fc465dc477953ffd29c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867277.64/warc/63f089a2490f7dfe6adb6890919d80ff7a34d022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867277.64/warc/64432ca015a3cfab80d613193b2e2b5ed40d8c19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867277.64/warc/9ee0e6bb21ef3b8956fb3d5b5aa7bb8f7ae950ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867277.64/warc/c321c843b72a1d43a94e02780115ad5541a9a431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/0630008505852f98562f973ddac8b56ad737b3ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/7b37019db1c547359f232eb7fe12bdb9a8b75ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/a00ab411864a90ba9b3cb49f4b051adef3d5bd1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/c489c28bedf99489bcc4026498e8fe20e9edd760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/c6340e263508734c7adc90ab5a55683674654f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867309.73/warc/ccb4e9bf0738b070783678e32d7ebaf3e7105cc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867311.83/warc/142f688f6e2fdbd0dd1ac7d689579f9f93e808bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867311.83/warc/644982b84ca62307cfd2d56a03723b1da77dd59c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867311.83/warc/6689a47150b7d3b5ce0e908b95aaf40a857487e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867311.83/warc/7faa8cb6a9841b75b6093065b04a74a028fa482a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867311.83/warc/8e560458feb9fe6e907a05e0d25ee27643ee62c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/14fbcab1fd1efdf9e4edd51baea0140169fb9910.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/32e54094ba419d618f6faa74f4e5512a0e857318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/54da2bcce534389a8ec2a75f901b7a0975346115.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/9b6debcb4db6734da7aac98b95e50ebfdd563865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/9c0c3a182cd22d31f6613f1031c00885b77ce66c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867374.98/warc/ed357f7c842d12c1f05a6d08fa48237a412702ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/7f1ebc225ee407733f7ba8fbcb3285751b8bf3ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/ae227b710093591aaaa8b5454183f633241566c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/ae2703313c9fe24d54df465814db01ac843c6a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/b8c2865ffa2bf6c4ec8fcd7418baa037d0c3a818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/e585b092a5684d3e4513d2ee11b41599d0cc0aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867416.82/warc/f0095358d66ba7bcb070b8d053e81bbc99b3d264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.71/warc/26bc81e2f408e22b028bc6e6800c76f4f2c3cee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.71/warc/3bb9f557f2d514a32854bd5981d28f36033aa54f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.71/warc/563b338542276170c2b86f0ec08b8273390a301a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.71/warc/7db5ccaa99221a9975094067498b78e7ffa14b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.71/warc/bff87a4724a0a25f7b163cbd80a31f69afc6fdcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/2f2f9f646dcf6066732e7af4955fd645dbb65fc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/9956a3815f8731d248c28aa10d1de88639cd1f5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/9ab658ec3f614311bc8f5f1861eae924050257d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/ad60ae9f8a8afdede12f3564dc8efe9d785c4695.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/c8286b2fe39eb357e8987b39fde272748b9d90f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867417.75/warc/dce3705e202b2411917c86ac921f0d57452a5b76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867559.54/warc/6847de9478c46211f3afc5fe0ad18953a23fad90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867559.54/warc/92eb13b7c199b36372daf757fae2c8b82bb97aac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867559.54/warc/d840107b12fe8d846b07a41baa76b0223782c459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867559.54/warc/e0424c25698e4c20c65d3100268352a8bce94707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867559.54/warc/e915e07d7ebb799c5c337757046be30c853afbe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/01aced303feb4b7ead009fe0ff327edd77269e75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/3ff9869350b8444584bcab71564a50b9868bb9ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/5c6ad8d683d102ee4952b9ac99674ad45d9778e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/9e404a1aebacaf866c0456be888c5861935d709e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/c308188fdc2ddc3ad6703f7af4bf4d5479b6fc01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867841.63/warc/e3b406f0a94a305d82e9ec5e5a7a026a0a53d565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/311312dfb093b0ae34032b2d85a0fbbcdc2242c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/6c5e5876ce5afa4b9870bf23ce8a635a8504e67c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/b59967c70e7498a3e6131efdf161d57c14d61b40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/c7a30cb03a02f7794924da22bb2e0f66cdc25155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/ed8e50e788f3b8879fe7ade9682a5deeec47356b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867859.88/warc/f0859628caa3cac4317ecb63b11573a95e6455a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867904.94/warc/2c32f8adadace0990b4af6c4c900d40f3fa6d2c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867904.94/warc/4575e49e0169937f3b26d054ff1b07adb6cc13ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867904.94/warc/5c4550df568ce30047ef92ebca644a364c5416d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867904.94/warc/708a07452917e3c1f726f63320c6a76811b08eeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867904.94/warc/9e856b5ec671190ba661ef883f07ebabcac1d720.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/00430900400f06a8e74e9ad47559b11e2f3d3ad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/004bbd49b6f5251b0c847d90d60917281c4f0539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/03aa265997844d9bd8dbdd2a059ab3ee4c21aa51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/5dc47c2acc76a924e4342eba9ecfe99104ab4819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/7b19276b8b5d9dbf71ea7fb7dd85db34de57c5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867949.10/warc/faaf4c17db498b9571b188cb3c593c96ce8e4fc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867977.85/warc/36ffc296f37b4d1a8b18b6ba9577be498e0c5886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867977.85/warc/6bd4e3f64e989e80810384222848c741bde42fc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867977.85/warc/b3071b6f1696025c1b98d1a28a1f8f4db8433bd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867977.85/warc/c7c1936aeae5220e456e538935a6f313033936de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867977.85/warc/fb44eb7997b06afdec3b69f0c1e3240042ee3160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/2b6bd55f09a9f9f9853869d06a55b30d2173a81a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/399f0e275e47f1e82018233460010c4298da71a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/6a6a65af2fa7a4e7b0d3e2e5fb14c8b6ab25fd93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/aff7efec011779e10be9d8171f20a4243f5f1bbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/de1362d86fcb75abea4982577290863a0ebde5b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794867995.55/warc/f58b51db35490bf9b8d5025ffaebd19757620717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/048dc3ceacfb3190a8bed4d0f40773e01460132c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/198db49a0452c8ffe5cbd81f39cc72ffe97bd7bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/630cb1e33f657e0aab6ecc57f28e9168ba05fb3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/c121173a93caa74248c4c6aeb9918af96567f64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/d08303219654d297230ed0f565cb84de338a3326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868003.97/warc/feaa4359ea76109f636c344b0cececa64c3e4490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868132.80/warc/144036ff537efe0a4a289a67c6fb5fa04b1c7165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868132.80/warc/168b9d7c74b1be15ff7cf945bca34f3decc08fd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868132.80/warc/6e24d737f3ef441a5999e8fb19aa0847bb688b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868132.80/warc/cc7f90549c33e130189fbede1dc15a0f77b66136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868132.80/warc/ec3f7b5f03064841d579f2fe6a02580dbff2761f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/0bc2b6303a815f885fdfbc108770145ebe48d4da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/0f16552b574942a4253695cb99807ac283439fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/40b2066284fbee894d2c4aab12ceba28d761eafe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/7630b878b4e6f989925cb82482ae7543cd84d3a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/dc3438407bab49afebb4a5a9efaf5a9b0ae98ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868239.93/warc/fded57500b6bed73c03cfb4d7cf578e344184d00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868248.78/warc/4405f58603b7e7dcff2781d7e8a5216d159624c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868248.78/warc/4cddf19008b1c39b399566e6a786847e1c346ea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868248.78/warc/782e1eb10d8f66961650ae575d87b49c31b53d40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868248.78/warc/9bfabd8182fd5aedde25faf020dc025557a916eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868248.78/warc/ee4b864b97ee285ea4a9c8591882402c261382c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/05fa2f501ddfc1f8ed019edb62aadc593752c0d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/0a70ceddd90e0cfdb6d011fb8ecc4145206becf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/0e4d3f2bebb942b288e0db372082d5332f9d97ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/11a92a4d467c20655ebbbaa9e9a6ce76f66191b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/9daae0a3b2d457c7da9b86164b7a0f98a63956b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794868316.29/warc/af87ad34e56f40bbeb0374b394a2bd74e85e736f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/10a094862de7964912b0e78443fb98f3dc169976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/21d9d217cdc485c8782064cc2600407909e8d617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/22da84be8621e47ef7eddc5e4cfa35b46f1fe139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/4343fb08d567a552a5da9f92780c61ea96c33e9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/605ed4953772c7f55ae4a2d15f949a65db8d0f0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869272.81/warc/7d2044f057d1fe3c3a24ca789bca0d586bdc6d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869732.36/warc/21205a6dec16c8df124812a42eb70f0b885696f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869732.36/warc/8594abd5502f5d2368e89f19c95884ef6d841631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869732.36/warc/bba968b7e1f3733f187109fece06f7e09f408bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869732.36/warc/ea4d77c09ac1e20bb054f936c3800cb5564bc57c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794869732.36/warc/edadca870fb522e3c46949cfbe99f729dd24f5f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/0754f2e2846cf723c0629ee5e0471bacdd709121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/1369f65d0c55431b6681a4c1d878f342608dbf1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/892bee52ee2512f1eb974f28cb9ccb33a1333a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/c6bf4610d76544e0f5ed08e4e43b67455490c47f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/c8f91002f77c8b346d246b22d7b731d2a9846b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870082.90/warc/d60c4950007d8e228a12df6c70d93c5cb43cc255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870470.67/warc/29306807b6ed9de05f3cb324d39251309223e3ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870470.67/warc/4e5f0c979ecf09f981e3b2a3f2ee21b96b7411d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870470.67/warc/59fff4330f724b4e55772eb106f347eb150d9b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870470.67/warc/76457ce9ec52faeaff31a866d6cbd876ffadc5b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870470.67/warc/dd5957b8406f2ecfa45a5eb085a9d9de97c14727.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/1d90843d80dc2a8c50d7e18130b1aaf907de79ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/337b2d993533cf6dd7c9d27a01fbd4ad6b983afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/55cec946a3cb644ffbeecdef7b220322a36b12cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/cd6a1db0cd823084546a3604f0cfc0b07f4b1b6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/e4c0f195b877d18db09a3b67a01317aa2fd7361a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870497.66/warc/ebf861b6addc76c091ed8c4f264de996a71f53a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/113e9331eacf9715b9ffe02c489dd5f7d27894f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/1cbb08e9ecdc7d5f7dc69e7616a9b5a1b06e753c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/517011e252edee02720623cf60927235726940f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/6aaee9732f667c9f7e50509e6af50de996d22386.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/a780a96f6d27d9dc60f9a77965dd500b06c976f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870604.65/warc/ba41010f3bfd2963dfab12db33c18182c3b19b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870771.86/warc/402cdb9ce74c29aece449a8a719f8fb934ef8abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870771.86/warc/631966956a89301d60790de980c4375c8bd9873b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870771.86/warc/8018fd0b6ac0b97850771ff2682be14e060458f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870771.86/warc/df870987f032327018b3bf81f9e3ddb25ff10b7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794870771.86/warc/f2b3cb2d968dc5fc9693ed59e80c2cc2804d7034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/2d54f1cbc01f6c16db02ed3865b3edcbabe52151.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/3a2d975694978ad122806bdb7ecdece8b16c177e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/754b0ae43baf8f74a76a20b728d5eaec2afe00b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/78a5d0887e105c57c6cad143776b558ecb1f4564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/cbb199c0650caea005e7256cbed87b0ce5ebf6be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794871918.99/warc/df5b277454487e10519b0b990e3d70a445278693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872114.89/warc/386f3fd289ff2064cdf1d60a4d632bbdd8fec67b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872114.89/warc/3f00a99afd5d980775fce27d4eca6eb86a0b1831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872114.89/warc/9ca14283ac944393686527918749a5fd8b82597b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872114.89/warc/b322100da41345ae6222a81120b788666f641938.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872114.89/warc/f189857b84e075fb1850b7605a86a85008022368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/323b00d338fce95c3236b805443624d1705267c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/6890854777f2b63f69da19a9ab3d59ccd791cd9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/6d3fc1fc70d0a4d47cfcc3fbbb7ccf221df5cd14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/825ddec73d9fe51c19c4ab40154a3da948955435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/870a7747bef6154c95cd75b2b09991feac4cd44c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1526794872766.96/warc/d0bf00442d0eecd5eb4e8f5f017e1f39c2d1f37d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/158f28a0afa64fffc70c0133a39c66aec29211ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/17009c45c85b30c7cca304302902b75d9f9be828.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/1732754e2c604fee5aef8a3bb33f134deb1cdca5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/22b21b26bc1511c48ebaf4ec95071a0abeb31aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/2303a56bee5fe0bde2d9088dc0a2384721ab7b90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/231deea4208c31ea436ca923166d771d7a2b3b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/259658c25fc91547927813e40ca66b95f03d4721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/28a6e53b0d8dd618329a275e98ecc42328ae00ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/302049d86e451f5f2fd96e1d7a51ae8228777d29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/32aa6572b2972cff3dc4ab39aadb430339a99576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/332c6113f0d6cf17bcde235bc455674e35f7b2c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/3e1cd032c231f7ca3005d6f31d993d2136aef532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/3f46c9d889a64e1f8de1ab5d704e900edc1d27f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/422b655f8d1c061981ff8f117ab434fc259172c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/464e03a9dab026901e1f6727f453dc92847e4f30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/4be5a7667c45c9c85c3342b9869c65655feff1b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/4faf32e40402fe19212cfc0ea0eaf2fc166464cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/4fd7db018e509a2424d77237486246c87e9b41e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/519c562ca220fbb8c425df4a5bfa43b9dfcb5b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/5273600d845352d15ecb5a73b7ff8364179ab190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/55dbf13b9e88b7c5dd9da3db543d4f00e13ad82d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/5b5b2f0e9226acc8af5307f84344c4a8f1c7a980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/5ce0559c275239cbeed958c3d9d13f119eaf9f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/5f055ffdd0138c6c4c5770294c5c8717844113a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/619b1226b81ebb61c6e6e6794334cde442c87be4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/623ffe46197bdfd1422de792bf55f7bfd4c2a216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/6a694f21749480ef4b93d17dcbe3a9589797cb84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/6d3eb6469d8562232ff429852d75a7552dcbd70e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/7155ac3ca88dbcd1bdb33a18c91d6b2a1e554fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/72d29a1160afe21e19cf88b91cda3dda2adaaf22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/74b0e172f59f2cb18d45cd9b495807dce7f110bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/76f8365caf68a2a1b0150a0747c6d98be55cb8c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/7b645de28b3f61ed37c57291efba1621bc0efc54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/7beec0e1c00ca95c3dfe480f735e6c274f983293.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/7f6b84ba66ef91b1287ae0ba382f575471353f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8162025a340119331350a68b01fbc0a5caea01a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/819264d9eab282f8ddf3d60df2a92b640a0b307e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/82aefd8af573356ca1e683756f070f4243eb8467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/863aa200db3a617619f7a141c1d4d88bf98a7ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8b0f38856b04fc1c0a5b79e1d4e02dee2e5f1e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8c9bf6ae0f943a19a8c364bf3b73f37d2311c676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8d59e82be494dba2b37d2c21a65d9385ff915280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8e64d84e86dde8af9dd9a5eebb4bd8deacff7f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/8f2345e053931e6a991ecb7b239a769158e30f7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/9102cd8b58064e39f3d5d05ec99793b612fc964e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/94bc354730114d0cacdfc29abfe0630f6d47c4a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/9d85a1aca7f6e6d9d94ae4fdecd4be35c7576b49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/9eb09af4d4dd5631bce75e4e18ddf16bbbdd90f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/a50b71ff0c6f75fac5892eccd27a25c00241c82e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/a52515f115dedd34eb975d3021c0e3f19174b701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/a62cc9b00c47ffbf8a9b0087ef995c2c52bddcbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/aa7855e387193151ecf9cd385efe0fbb70d967db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/ab5d753a18c89229f5711532a520e6182e0dd36d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/b5a3938a7fa3b429021086bb7e65cd06b19be4e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/be3453ea753b86b47e38707d39f0413860885855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/bfb9b7e2abe470b4595c1edf01371d8063094d25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/bfc626e0c7649b321756bf564e151f9cdddba082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/c4d447c8ba7b5d05166b240bef77903f711775d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/c5e294f1633d7de169272a62258e6ddc04d847f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/c6fef54070c237488fa3ca91684ca8894d3bdad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/c75d4abb2ae956c6d1abfc5de547e6c42d869dee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/c84889d15de5ffdcc9aa6f158ed09ec3e0156d54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/caff15eb846a3278869cffef134adae57ca99341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/d2ca3523f923277658a83d1fa0e0287e8cb8cf2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/d2e63ced7164a73dcf3f716249b82c6b695f0572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/dafec31d08af3c6b91d79a7cc8d3b3c22d5a1aec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/dd902e0b6728bca859cc7b7d6d41c95b86b3280c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/e214460450cdf4c4acf4f5d8b065f1b647c78bd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/e7eb29532479a78538b25edd64ddc6a6146b42b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/eedea2a08c83a62f642251c564dd73d32ac5cbe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/efeb490f7858f3dc20da0b73b471d9762ea264d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/f18c2e565ff1037e9249720c3fdc0f05a7c26d97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/f7217c67e50e019bd0012ce46f9ffcd87392af44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/f92ccc5dc0bb7eef91dacf94a8ff5d771fbbc37b.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-22 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2018-22/mix/input" + output: "/tmp/cccc-CC-MAIN-2018-22/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2018-26.yaml b/configs/cccc/mixer/CC-MAIN-2018-26.yaml new file mode 100644 index 00000000..396c743f --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2018-26.yaml @@ -0,0 +1,754 @@ + +streams: + - name: cccc-CC-MAIN-2018-26 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/01373d8f83d03a9092d7b9d1a28234058e023c3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/047bf4bc1b30a1028be790f8d8d86c5c020a8de3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/06508ee504ef9faf5919342db17f05ff4c061467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1106b6ca321c4c3f204c0110544e9526eb1ab0b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/7a76e9ad7fd4384a6e3f1446c4d1c579143ed238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/b5cfee863b4ae5e18c8fedb7602f07fd4144ffa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/ba74a77bc89efa8277cbb3a9b19c9e07c156feb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/cf13bae37745938c7ea8a35d073174c8ef783a1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/ebda20f10b4dd9a042f3f58f5f16213eb5889370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859766.6/warc/f52727d379745eb68fa3b724b1c875c0354edcd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859817.15/warc/162152b5ebdb11f4640f22b1d56b8eab5a996832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859817.15/warc/2e37254220cff5d7fd135a146d4cb4c90de79ca2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859817.15/warc/5cfb2d2f6546c21306d9d70e0bd16f677b4d312a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859817.15/warc/79a78a6a4c9b9797797f3af9e357c1b37668b2a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859817.15/warc/c34baa5cad993cbdaa3080957f49f6472077f01b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/1d7e0b337837f58a6916535a43ae7c29457397c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/43f1983e99c15be78e091460f5691d9e7e551ff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/76a9a90efc929fe3193ffe9b89f2532bd775ca07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/9b2b43eebeace55ebe0bddf2e7e128958415b00b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/be728fa5e59e85d8d34a69c6113680f9701bbd95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859904.56/warc/f239b94c6ff8f132a35af73741d48fe3545f98af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859923.59/warc/1b98a50fe4de991f342db4bda12681a6b7f3a162.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859923.59/warc/6d03c620eae1d1d267654cc7c5e04a2b71943068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859923.59/warc/b5d326cb963abf4ca7b4b776cd55557514c53ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859923.59/warc/b944d17214e6b798b9945675bebfbe6addfa670c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267859923.59/warc/cd9937bb0e687dc400eb062d983a25daa5e47460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/0920badbfdd4aaae57c0121bedfd1b3fa598d660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/3256ce344e61fb9688eed6ba92d6e485bd1f5d85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/ba256609a9bd4b35d03904dd5db7d011f955efc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/d3ea6cc25a3d68a0655d997be97a2bb7f086c328.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/d491335692cb059bcae739876a99a5edeab69fda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860041.64/warc/f11572574c037ef7233455e43e06cdf87ff1a83c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/023219af42f4f70f006a98a673eea6d591a59af7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/1a7948a29b01014d42afed38347bd1c4111f5319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/1c70f964403916bb27f03e56d589951dfa32a10c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/33fbb4470cf8f444eca5530afb9af06093d44dcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/6210eeb00e768e7def0d01f4dbd362c0bc5289db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.11/warc/9a4e69cbb133362f4286399a9007f000c1a2d7b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.13/warc/1bab2e802242ade445c59aaaa1ff031d1ef3c896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.13/warc/44bdda06490293546a677209e9c31b95ec3c661a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.13/warc/55380395db565e822c9d1cee634ebba79cdae79b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.13/warc/5ef432ece6c3e39318eb23c4355510199102e034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860089.13/warc/e9c48cee2e9e6ad6f8726419cff8492fdabc57d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/0c93e279927a981fbb5a3ac1b4864b0e8ab6c02d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/3addacba83280e7d14b2a27e3ae5a514215726cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/466ed444024959d7b12aeaa8489627f2016c19a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/86387c623f546467e51bfbc032a5ebb63e4c284b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/f179218e3d3c0329ac6fe7804cd28fd3feb235fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860168.62/warc/f34428f327b5d716eeb286f7c28833487b436f34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860557.7/warc/5adacef7d587eb79cfd7f052729d7d4bc6d38070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860557.7/warc/7afb1ca41a98a1c589cf802a9d28166ca89c2142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860557.7/warc/876639d235b0cc42cf082f6a75990a93d50a6d70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860557.7/warc/bc68729ed4764bbc775eb313c1a2bd3dfe91f2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860557.7/warc/d65a1aec5ac5663f7d7033e6167532e501d13802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/1816436a4e71ea2440e6bcf11a55ee6e75b68e5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/41eb8ecfe4adf62a2eb897cb9c223df02222fdb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/76f2fa9b40ef51777b1183353bab157fd6a04685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/cf8d216d8326e63902a9ec496ae46cba1920be53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/dec8aae01a2cb2417214731ed0965008f079782b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860570.57/warc/e4533e1386108a96432e11952bea8cd2f40244d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/38e254e784f0ba03f8122f9f45518833de4c4f97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/47e7562207d8690ac42b8afc0fe8d00d873ebd5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/4bb92c40a2e19c115804c4c11650c380b84116d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/6eace2ec814fa6577228ea5161174fefe0b6c506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/f4e6b63be964540794983c2764508ffa86271da9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860684.14/warc/fa56c7d6e817c352506aaa68ee5a374c534b4022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860776.63/warc/0d775b9dd38cd5f78b7a70290cabfdf70d8e1712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860776.63/warc/403d8420ad1fda71381d648a621e0b7efa387ad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860776.63/warc/48881282df6cdc7c1fc56ebf70188c5d2269abf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860776.63/warc/6c056e07366552678e962a5c733c7dbbf7849c43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267860776.63/warc/9d5b4f7e8766aaf593600d7db5dde3ebabe91e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/1ba987903c7b6f9bc1f6826c437ae72d9ceb10ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/43a73a7a7e40d007ea65f6553f638130b9e414ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/58fe5335402dba1768091a4d3db786e50d8849e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/6fa0ec4cbf598ce7138461268fb3d5bb8b2acdd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/9709fb266702ae989e60f33e984d127c4d8a61f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861163.5/warc/ccdbbe714bf90b3ccb483cc976ac23016489b761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861456.51/warc/3c0982c16ae70e21cff276fd774cba3ec5c671d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861456.51/warc/4eb3b76596add65c6f78b8f2ff4064183a09e9b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861456.51/warc/8ddcecd015d8e79fc8ea2a6e112630ed108455f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861456.51/warc/aefbbd04b52bd95c350a26d6e75b24e7b14a76a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861456.51/warc/caf88dba69c7f04ddbc411e737bd485aaa058eda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/073a02a6cdcb3e5bc20a4585ce10536860ce3d4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/40b913d97f746df29bff904a49a85663ae3b6969.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/4305c5e177739545bb392066d2102ead6c856967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/54acecad70099457db78617802afc36eb575ff19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/69c4499a0c0128d0cd85d78088c4914cfa11a273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861641.66/warc/9df7223be0986eb1254ca1d2cec17c6668601b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/109b93539c4a44f49c008fba279e4e14c98a0879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/2129bc767cb0ce3d1ec6c8d0d1094a8bee902ce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/5183f53c76cda5731c964ca2764d79e2c019b99d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/816af0e0d779c75cbac7911caf7663c82b3804e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/db40dca1ac353accf38930835dfca8b6856fc872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861752.19/warc/e8205d895e49894308156d2c89a3c464f26772e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861899.65/warc/06c6402f27fd629ad29076485f89386c05620669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861899.65/warc/3ff2de261caa2ce298fb60ad1f8079abbef5f9d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861899.65/warc/503f54fd517bd772846873c9f3a6841cdf2111e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861899.65/warc/82cc3d71d06c43caa425a3dabfce51bb4a0cd4d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861899.65/warc/a756d40429075c9f303b916930ae7ef16a631966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/155da8721643f128fc49c6a38d98eb1006cff9b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/4288a02d7b0804beefeea53b182d9373c4b007d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/72cc59abc9845715f29136dc356ddb93b89d7bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/731daa8c03f94c5c9673df734beae6d994cca497.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/8dd42952c52789e2ad16171426c014dfc007964b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861980.33/warc/cbb80935c6ed162eac6d54a68eef96214a96138b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861981.50/warc/21c849fae6d0cd1569d20730d8de1bca3c4a363e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861981.50/warc/4a9feffdb22c8aa2a362f7f7bd13cdaddc20dbd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861981.50/warc/7203185db6528049eea67987686da0a7623aea21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861981.50/warc/c6c4d731b42d8789c339d3d717f57662a26696a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267861981.50/warc/f2ddc51e5543db9216aaf5438e863776d8903013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/081bd5e1e233681bb2bcb8c533ccf08e4f107055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/0df01172a57798ce307c68ad979aa527d35928a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/819fba1dc9c9eda62cb5a767ff0aa9c63e66aadc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/ad0447bad4f6b3af2002553908dedd3beebfa73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/c136f045a2c64e7a6d7df0e7987a228c92251657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862248.4/warc/df4232ffe488c1f89bece245b6fea9cadce79415.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/398d1e649c4b88a40bad6463000ed5539b4c56a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/4db156faa4bebec0889d3f60311e88e3518ae5ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/67e49fd463cdd8dfd71d1a6b4000df7a57c133ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/7df0b1a38763046ef85c1f64c2bef78dafe30c0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/99ea82d42871d7873ac063dc7496557f05a2bd43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267862929.10/warc/d5c99bbb7059a9b1d0fa8eb8461655d44bca00b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863043.35/warc/0046cfce9b0fed21921f70a567535b463b24ee85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863043.35/warc/03fa9b729025c8c8898b2d3220b26f0e0fa6f0d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863043.35/warc/0a2be26d85a1b5f62d9285f94f88995738da286c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863043.35/warc/17002553012d30a9853e3361c3588908171dc4e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863043.35/warc/2caa93f44dc615d6dfd7845ce1aab7a4aa5e9f70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/2dd9b504802d1637b6f2acd5b2f786805dad9cc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/3eb7d68859262a1c8ad885ba8e19d6e187a16c10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/42cf82d71b99b9027a238be9185cfe24dc68c6ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/6aa2da9a6205f4a0e9e70d0c9ffb4623f4c63296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/9a4d4b4d37ad3a3e1d2a1bc6e737a47e1b9d82b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863100.8/warc/de323fbaf3be5996d7b6d89b193d1600dccc7fcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863109.60/warc/00c2226b5a4e33585ab3cbca8846464c57d1c610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863109.60/warc/173335dca0dcb3b48ee71da040496cb3cfacfc23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863109.60/warc/924b9d0dbdc4a6fa90d317844d81d5d3ecee2379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863109.60/warc/c4d0e45b19da742e24cffcab11099cc2db4c2325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863109.60/warc/ff203c231b178777eea3ac085d8c52b40075c5cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/269164e0897ce5609474aa33ce5e07c9e54da895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/6d0685701728f67c8c3c685c6c78bdc524ddac61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/6d9f0e6ac5152ec0b8f3841a48ef05bdd4e188f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/aebb1bad0c772e8aad46523d867068308c5357ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/b4fe1edce9dedb98926295eafe133ff610bcd9b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863119.34/warc/db87e455d67b2a1fe6bf249acfba6e7b4873b9bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/22aaeaae2a7475410a78ce84ba4dbb3c87c6dd83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/44f3015f73237a9292e53a55dc5f9020798226a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/465da1e7788c28772b8cdb30e64b23e2b1504fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/46b9b01edd9fe8c0c7467f8f28b6eb0f026be876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/947e51a309f00472974876b9980cfde3568e5993.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863206.9/warc/a5502d89a1225c78441ffbd3380c3111dd3648b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863259.12/warc/5e594964903c22543731cb803af3f0e9714cfab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863259.12/warc/5eaf48c9ef6ba75efca8c06f3ee647443423c3a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863259.12/warc/712700dd59d5e07abb406c862ed23f1cf05477b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863259.12/warc/91db54d60d4741dbb0497f593d030ab1af1f4a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863259.12/warc/c4f3353fd867e491f1676170f32e6a27c2d505cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/2c5ba0be2ba4c9a18f65534d199f557c68944907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/3f39467795294e97516e6a9937154dded6202906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/4bee4c3e2e624a2809613fc6ac13591c837b952e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/66b3d2951b5cb34517dff274b8c3e9af23f0869b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/903ea7aa811a77f3143fde8e01190379dc8c4899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863407.58/warc/cff07a6126a64630701026094bc7d2996b177796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863411.67/warc/46053f3e753700e7f3fc0a323b8ee4d3d36cf7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863411.67/warc/70cfb3769d01b515e14ef7524c94dc89fff8f576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863411.67/warc/9a802b58e6a360ff468b9ddaf1e9c1874e220aee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863411.67/warc/a45edd425b190be68eb0ddb268e1b4c6932fdbee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863411.67/warc/ff6e6cff627c46a1c01a59786fa66c5e4795563d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/655ab92978d6a67a69dd2b4b83d9796c0e2e1081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/b293b26336be4d785b3da4d5cc2778e2de570467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/c7d32219358e09baaf2bc3686d3d8f61651204b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/cc5374ce98a626b206e31be0ba0061347a5cb2af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/e51f757030b96cfa9395b664fbfc405dedf394ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863463.3/warc/f1ebbfc23c1fc14d99a2ae3239301fd7f374ad3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/0d9532f1b998c30dacb3539a297751e40d29ccc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/4ffff5ae229a3878c0b9f3344b24f218ecee3eb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/697484d0fc3163a8c5d9a12bd23ee00199975637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/6ab855cbe1fe8b40f0a02d30819da89f2005b018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/85adeb5b205680c9dcbf66b17901563c86fd6891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863489.85/warc/ff151b390d1b6b2fdc04a2cd53e25ad164a31de2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863516.21/warc/2bdca2b81f1ce462544f9faaf3bfe7b56c6d650a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863516.21/warc/5ca2055b3772ab11957eafeaf4c3b7b89c020abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863516.21/warc/5d2be1c3820c716cec52b6f6566e2b9fbb07014e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863516.21/warc/d189b7e0a03ed32d8e591998d450a1b7b90da0c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863516.21/warc/e3c0129dde933e4ab059b66f8ee82da41168c5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/240ec41879235118fa2a2a7e6f19e3a2ec096a61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/49515d29a0146bfcfb220ed011898c312a97c134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/4af426a3d2b4b74c5c8a431c7d7f433fdd541faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/86b465702d28a87a52fe769c49fb12cc9b1d4099.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/cc5864359ce4035b7333ba5d6a84687d16fe9a4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863518.39/warc/da9b6ebf30b76184f79e94714f9a1ba82cfb4581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863519.49/warc/90baab6a04400a9384e8bf610cb60b104dbe31b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863519.49/warc/d10fcad4ffc431cd28729b2822e7e520b785a997.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863519.49/warc/e3cdb73f7b7315cde83f035df43336e4d1ca305f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863519.49/warc/e7aaddaa35587f10d025bbadcc647e95d0b7bf0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863519.49/warc/f33f3ea1f121cbafe848ebc1b108379f79e6fbaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/32887e8abb1fb83190472442e888187a681fcbae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/3cdeeece0997b5182bd70cf1050e7167edf2e2d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/5262befa54c39758655ea52884d3c3a4f2e293c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/68566bf58a565a3bbaf4195176983d0486d74a86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/899dac9e5fade4eaea6ddf4dce54e6b75653361e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863650.42/warc/fbe3c3141c21dcc663ea400ef0d523a985b6323b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/743e0250596e42069d3b57a5301b0931209f9755.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/9d830945debd7232bc0b54b615c7ea6722cbb854.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/a0ec920e5211a3daa4eacc7ff65e256dbdda78f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/b91b0dceafcef1e362c8589c96daa34288a4c91f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/ded0f820791cd5fdefc3a11209563a83fa145b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863830.1/warc/efaadc2fe227a0b150d8396b97a36e5a51796624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863834.46/warc/1ef28e5ecf065e611925e5ccb7de4fae8dc7cafa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863834.46/warc/1f29dc02895d6c6dd963501961d7412d4c5512dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863834.46/warc/7e1d9ee8a3916ed2210bb3001a54f10f69c4fc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863834.46/warc/a4c5b87676a0c0384426dc5cbbf2c19cd27d38c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863834.46/warc/d84794a7264010afbcc2bfa2dab9fb81bad83ad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/123605d186b03e78f5ec9d5042c53560e3af6808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/56d2bc16d1e55e860f4e9bd8e5c513d246429836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/6326ef49ccb70393a5d2ff2bb5f0b7cdd868c84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/da96aa7f5229363cf5de8daba84306acee467274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/e5097bfd65b55d7e9a05a7735e3308f4235d0b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863886.72/warc/f417fe51d3f03cd7bf409829b9fcab32d910d3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863939.76/warc/7c088f46a12e94440de7a22b64fe3de991abbbc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863939.76/warc/92631fd1c9c06f6e6bb3e521f71c25fb04504bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863939.76/warc/92f87b3867960eecd4d2bea3d14dd19b08256165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863939.76/warc/a9c5821e233e839af082745e00b610ba91c9af4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863939.76/warc/baae38147b4729193a8fed25af5e5610653cb74f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/031d2ac53167ca98c9b1da94fa3c94aeb5a1ed57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/17e443df835b200335fe2e45e85576731ac99d90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/1a1bd58d038bfc22df5b7fcbf9c03fefbdeacd49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/79dc77beeabb4b19d274b54b8a22240c2ad266db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/b7768ed44b1be2dd6250fd74704a5b946d8aed8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267863980.55/warc/bf101d8054ca59bb8089a231e5a28a58e9357d4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/00ee5f4b99a6428a125c17511b58d18b076cf29b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/391f8506d323f9543e7a0354ccfe74e7b1d5d8ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/89060b504803a9ef337f67b1bd7614ad069f16d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/8c1c1914cee7f76c191ff65f234c8b69b0a17e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/cab6b5ad85555d79f7bd031c1479c42b9268ee84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864019.29/warc/fa2a8988e1de97a6b3770d536c5843112ef26416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864022.18/warc/19f52b7440f7b153c291104ef98b2021170dab65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864022.18/warc/1cd8ecf3705353806ef59c2630a32d33ae87c299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864022.18/warc/64b31e560483547a84f31aaeeda3872d02782aec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864022.18/warc/a42ca2bc26215944617cbb2edbcca5145de12424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864022.18/warc/b726dccb6b0b7a527145127fe35576c95d7ff853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/791f622e623d3c8b317cac56b3f0817618143c64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/9bf1a7e06647834653c0bf158533a76cdb1a15d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/a0e8070c38ae0195c46a12c2edac457d48a0c0f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/abe2d9f7e339d88604bee2b5b4e371ffe6de9555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/d87f7726fef55da853a2f92a2094885fcd4f4878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864039.24/warc/dddbf24bdc9f9023fc32659217b11eaf6830e9d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864110.40/warc/08d9e9cdb1d88999664d4d9b50c86e78d754f0a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864110.40/warc/9457fceaf9ae92f989456a69c0729ccc39acd3d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864110.40/warc/d622cd8c0971f102baa39e34d3d15055ee598296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864110.40/warc/e444a39d0f531b510e4366eb4662e76d406b592e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864110.40/warc/e5c9aac87d32fd52102c2efe0f6293debfda4b30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/19d4ea3bc8d813cfd3700b44bf9d152b73131f4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/58c9604e04423f72690914110072275fc69a945d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/8785574ef753bfb9c1fcabaa1e9afd3945d00ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/a187ebe7de560643200249c46e6cd2c80a025644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/e8171bc8d68717618c7bc5bfbbb0a5599cea1f8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864139.22/warc/ee31b5c21ac35caa21b8275c3610b4d3b8fe8849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/2df320e852207472e61d3aaa431baa4632421de7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/8435254ac4d23cc11f135477fab843557aea5aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/98a6fbc735f39828ab5b27bd6a95f84964ab4a42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/b598dc3255791b2b25103168d1a05b79e2fb0739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/f5d23e6bbfaa4245487b818c085f454bfc11edf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864148.93/warc/fc0395bf9308b1a3ac44ba3c0638d9fbfd3ae18d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864172.45/warc/341f6fed89216eb78e31549023bbb449ca8a801e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864172.45/warc/6a667946e18e2e87207995bfa55f87a17244aeda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864172.45/warc/c3b0b7d182b782d7ee16e54beef092d9accc788d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864172.45/warc/e4c7928741b3e37abc62ef29127685404ad910f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864172.45/warc/f7c78af331d74f371976079dc3bb122abc087e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/0a6b790e518cf641a79578bc5dfa25a1cd3c0f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/13f3e0008dcab3e6dad60fdae85d344bc4565d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/1895d9e53c42d362562758757097afdb5adeda62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/2d4a03e10960c69014612dbb0b7e58eefe73abe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/a394cc38aabd6d8001d6c979e7d70bee5d40faf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864191.74/warc/fd0705a2180af2384e861f2c6e411891ffe97ea5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864256.26/warc/16e5d28a466fe251cbeac8958137fdd0e164e2f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864256.26/warc/73c76919543b9067ca0f757882351d538ac4b1d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864256.26/warc/7bba56d68ed697a1f032409137f6c93675009cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864256.26/warc/7f23d458c0c7d4747b9146b1c770ac8679edb5f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864256.26/warc/e50ca26d875bae8a302cb654b1a60f7777c786ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/0e1d494ac403b5b67d7c395e8f6c994c198d4b91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/1c7fd7e9a20f1bff4925df96187cfff0e8f2038d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/24db6c6384c85610c9ffab44fa6460fc5888401e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/79581abe63602b616843c4674545aa2022ab655b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/aee0609157b0e7c5324f6d22e08dd25f74e9ae33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864257.17/warc/e2dfd7ad6e289f687cf191bd75b91e3dca35157e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/966a28f5e14180671e1b99fb5159bac38f31271c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/b1b2272ed024fecfeecb9f2c2206fa65f250a999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/b65babb7fa1e99ac15b76ced46c59f766aa1555f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/da47c49d184ab34299b5fa696e2913e9c73d0716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/e6398200738c89469ae90556be5e70491e091f60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864300.98/warc/fb2d7c9e7d7cba981691470c8c1bfa208931fd68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864303.32/warc/0ae7f833c7ea82046aa1af62573e99fc577be13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864303.32/warc/45049e26cf0452890ac7d063dd71e87a1072de23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864303.32/warc/bc121be8793f1f7dddf651c6da0db1513928c46a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864303.32/warc/bc9af692b1294c45c63c9e0d5b58af66c8a29e6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864303.32/warc/e1836e041344b36198e19fd1366706a9b4c079da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/2c3aa9a73066172e7b870d8bab3cab189cc10c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/5d8b7dde7dc895eb80e54c8205dc3b0da73a76fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/86d5cfa93d9becae6afc1bb6b1e27e07d61c3078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/884362fcce18b988edba9051b9409620798ebb11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/c9c38fa08a4c398c6f6e34d832035748e59295a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864337.41/warc/ffcb1b958f7c29ab907854cb1164046eded37abc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864343.37/warc/24e077be492ca713f46cea18d8d4c821c2a68e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864343.37/warc/4a09b0480d3850c8153f5fcd8f6b9803c11d0678.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864343.37/warc/854e15bab6bd3db006c1eff4a0cbb41eec743f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864343.37/warc/c39f0a9401f114a014066391f8fc150903b1fb12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864343.37/warc/c3c1136d2d0b4b6b1cc437ee09ca9d324cd8f66d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/0155e7492d20bb70ee95101742e397e9b8c08675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/1849d6b2227199f693fcdded715a4e7ec6919935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/32154be98ab6b7030fbd7e1ffe8803565128bf72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/5de370f6a1ff909aeb4301c3277821c3b92e87aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/92c2e55f2130a4c7055aa9b54dd20e77bc490100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864354.27/warc/c308c82b911cfdedfeab1233354ab5b979b50a21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/055c0a19742cc528f4b048b2d659bfae5ba6e1bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/2204d2f1e6a05a3cf1d0a714a0889112f76b741e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/66b1856322bcb6d2489d631d1bd6170481a2f0c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/d62bd4d0baca2c57b1d207ef042908620aceab42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/e15d0b439f085c9e7ee193c64cd3c99a5a9d35a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864364.38/warc/ed37ba0be55380b9cff8aa9397072f776cfe8ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864387.54/warc/25e7b444442324fa015a396eabe9b83f2b12bffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864387.54/warc/2e60dde8df76c9c44b7a3ddea26077dca286f868.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864387.54/warc/439f59948c33aaa4308e1c464818aecf85ae22c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864387.54/warc/76379e97f05d4c6de9e38864773527c1fbb888a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864387.54/warc/85d47310ba53af31770ee8ce2dda275631f21412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/2792d84b83153282894b7565bb004dc8cf2c3736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/6170cbb471b24a6c3eb28c6337c59eecc74f78c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/84d791329e41fd9f360fe7bce6cef2e80f41ac20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/973dedb48f54872c8d92d1ff969408237de4ff16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/a5106622fd1c373a4142ece681b20eadaa4d4e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864391.61/warc/e66e5121d01293eca6adfbadec39f1356c5478c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864482.90/warc/4c902416774b608ddd7c0f02dcae81071854948f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864482.90/warc/b4418d542bc4d309446bb60ef3459b994acb0a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864482.90/warc/eba3657e433d7c088947a0be544211ee473a492c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864482.90/warc/ebe35f178ec49bbd5e2a4aeb4bb6459a6a138c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864482.90/warc/f314c3972c51be6b52f8d2ab3d3bdabfe6cf39b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/0035963c416f38ebb5d538ff1134c8fd9b6ff9f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/2bc48392cd023d346758d30ec303d3d9e718e350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/8543ec170a53e614ec6648a10c7f13d29043fe04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/cb46c7f0b432d98c69ac4213d7249f3538f2d942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/cec7c5800340d5192dce96f4aa43956e630825ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864546.30/warc/d7f02acae23a6a3b015bf583e80c23bf28e1bb46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/5a72d982d6fade707955be930802228e02d746c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/6c95cca594a7eadd7fd3338139cac67231441daa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/8d11b3b2bc47fc6301809c875df1d87eebb46e26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/9d30c27fe3ee73c1950fefbe59f34871edf89beb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/b347bccc5856d962c6d5e496756e2ebdc6442aba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864740.48/warc/d7f917556c96ced9553aa0f9c12aa3e723da1f2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864776.82/warc/12ad0b3b1c7e2f3c0be2778536fd4e9dea92e76a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864776.82/warc/14416decad2dd6a6b7ea11f13785311c4dc2e1df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864776.82/warc/150c0c0f822a1ca922d71daac3dea195b824757c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864776.82/warc/80bda9ad2b7352750423b252bf3d371c4e709240.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864776.82/warc/ac46ee4f10edcc551aa4376d4afdc1d49d859ad8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/7cdb9e3f948b205c342f301416e420b6cdec08ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/9b1f1947cbd5d25a2f64f386b580c59f90caaea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/a53b33d53873b704ed7e2eb68a2ee02866e237ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/b90eec93dbcd6a65eae3485b5ae78647ed53d021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/ed813240ece2fa4f9005c30a9f6b8445ae8b0ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864795.68/warc/fe97f8ae275ceb5a12e6346cffd82cf2037b3956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864822.44/warc/02986328f8f4e5d407018c5c688831a7ad336dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864822.44/warc/2086859bf3dd3be14d36457b096970cb1b638288.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864822.44/warc/4f41fe5d5a2863b50ccfe4d0159c1ddeaca8323c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864822.44/warc/a3fd527b5fb0adcf69d1dd76058f7884a3d24335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864822.44/warc/e8023900dd1efaf78b20ff5c3b71074f9bce0f5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/0c41405694ad993c207729b42821096c5fd20ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/777ea296853abb461deb123ca930efbe83d38780.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/a02310c8a63e8eee6a7c4c050966fa9600b0c490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/b160283f9aacf2002abf9e8af443568fc0ce187d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/c616bfe0cb85baad5bbeee3576766f94c494cafa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864848.47/warc/e680574c859130a1fab4b67d907743b1bce96164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/1ae25e84b196aa74a645643ee7df3e73db5c5b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/22f0513786d55bbcc3fa3f49c115677f03861cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/437570dcd52cb625e1e2ed6d601cae50c751708f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/c182debf35c1e774a0a8baec23b43e82e9e7a67b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/d745ef112736c07554c39a986e6664fd9ce263aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864919.43/warc/f0f5d8db7aa0deb7d836b85d954f5c796ff3182d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864940.31/warc/475fb181d9143791fca90a89865e22857763f33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864940.31/warc/573744af410ecd7d7519d99fef784dcfaeb66cc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864940.31/warc/7c1e5015fbea49fad99775bb2944c7a8854ded2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864940.31/warc/999f6ac9e786f21597dcac97c1c82b3d9041b6c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864940.31/warc/a36108a3c5ccdbcf0381154b4e9a94ef6171da08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/000b8e3bc9f14e8c7f6d4624da3d63469f841a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/0f92a5c2f5869753dece58bd2f9902ceada9d7c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/842cbf14f032ac13d8db8c5442be087b4a3efc69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/98ddfd95bd69ccc24c325d616efe1166c252cc5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/bb46cb69a7d637a460ff10a667c56caab6d42401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864943.28/warc/be9d01c335449f5a649a9c1d8207efffba875e7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864953.36/warc/753221e9e304d12fb4e9341c1a2d0323e57865e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864953.36/warc/84ba0cd492a91387c20f8a27a0b68d60fcd57ca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864953.36/warc/b5da281cdbd2f992c85b5ba310dbce50069939cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864953.36/warc/bb4c07d564f6c42b843540bbdec4370cb94a13f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864953.36/warc/ec3c9c7ee03a37806dcffadafe96cbf31f3983fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/64243888f23f08c249dd7a023a1b3ee04fec80b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/8836f4e82843f8d05415c9d490a093264b4d0622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/8870bc6e7b21b9a416c17dc3b7ae655fe4f2b110.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/c1be6c330f92c57f78ef296b241c00b28a8db7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/ddaa6e44049c612608cf62e9f8d5472e6ee2600f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864957.2/warc/f9766df2c1c7d7c0d91f01f09a5700821c8d4432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/2b12e39fb3539611da2d76c2eb29433e87af2b4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/37cc06375005f32b0c41102e7a07161d0dcee6bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/6fcd8821bbea28168223aa0d71b6b96acffd0e5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/7282c41267549b2e68b2736bca15eeee8da9208c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/9cc63a4b758aad8040df8ff1fab1d376aed3d4cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267864958.91/warc/f89eddcca79e1019b817182d78a68e613223cb8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865081.23/warc/095ff9ffb41dc6ded47c1655fa241a100c36055f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865081.23/warc/6fc81c322fbbba978cee35f1bd845ab792377065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865081.23/warc/c41d39c8eccda2aef39e11669eba1767a6112b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865081.23/warc/d90a21a90ed545859d4c0f27f6496aba3ed17df8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865081.23/warc/dab0059497b5e1372cd90ea370640cafa7fef838.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/1d023aac08f8537534b7ba956ff3dde13a47d661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/21520de2968b50cc30bb0dbe86251b11e81e8ffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/248313a2c2b48595aaba6d4aedfa20d56d6075c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/3557ab87efe4d6a90ad8bee9c4d9a7d49c2d89ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/52584719cb5a6c01a8f06ee3fecd7969c0f3f1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865098.25/warc/cc434b5f2111cc68b08f72094147b7dace40d4a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865145.53/warc/4a8143ea8a73765804e3a4938254de14ab6eccd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865145.53/warc/685b9a32083faeb187c922f88654c1de8ac4cdcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865145.53/warc/90e590226c03fa0133790597a067a0a75a97ad81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865145.53/warc/e221c1997286cd598d9c1575e7cfafbff5c10821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865145.53/warc/ffee9f866d7ef42478d8e800d721aa5605a5c40f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/1a5f601def63f3e7f361aa7add64ac4fc1f2f242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/42f0df2ea828fbc9f4b28b8cc8bd498a59753a20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/6cfa2bbf82fb3b435313ea353c068848d760d808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/6ed53bf7e13872d556045c177421e4353fd8768b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/9e27c698f0118b64edbee7fa520537df92fde46d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865181.83/warc/b53ed5d19efc9f264a568b87f3e560bd97243d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/73235ad9f176ade6db3cf7a288e61c5421680aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/781e51410ac572913775c0fba208b57c6699bbb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/9dd4c851734ae23607d0146b3e63af2dfcec8fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/ac50b95e0960f5a6c8a59c9c797d1bb3324499a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/b4877784e93b168ab6e83fa13a728e1ca31cf64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865250.0/warc/bd7080315f9e93f852a447783c03469f1f1bb1c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865438.16/warc/03faecba6011b8c23f50f323937219c7b7ce15e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865438.16/warc/5abba2bab7d9a21e39704c8798b242a6ad50b899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865438.16/warc/5ecb9a8c9732951d890dc04e4b20b1fd6776e4cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865438.16/warc/9b4b21825aa54b099134d452abc31f0283f32080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865438.16/warc/f7d0235a37c4a257fce5129978c1667f16d35765.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/1718f718516dde4824eac36446c3efff2d83f1a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/5b277a6d213fe14eb258a55f0d73957981005756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/7ec22ded47e6e40bb825877ff07afa05559e3de0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/ae3fc9497598c5406893e468429d0346d357e74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/bca77541deba12c34cb2f96b824264a341aaf749.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267865995.86/warc/ca37801be2702b4b5bef374e43ee6b66e6aaf375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866191.78/warc/47a7012845c8944f5c623cc82ae2dc04460bab68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866191.78/warc/7f2839dd072ac85adc9338321e53b4a5dfb266de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866191.78/warc/871d96e6a1add6cf9b5c479c63eb0a2dc6527337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866191.78/warc/87a7cf56a662d565ced9039aa3e78b4c4d654a23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866191.78/warc/a127f1e32bcc35ca20de747cf59a50dd804b6e54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/1202260296360b4ecf6a68c03667937b18e7d107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/49d352151421564284703c773bc4ee7488f8edd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/838c2adfff8e6f9244f6f19e24df0380f6046659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/c05821e3ddeec918bbf7fa5cf24e85d227d398c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/c8c28a41106c8b19d0529847e4b85627efb92329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866358.52/warc/f524ad8a23f71a2d664089d6e0812428374e87df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/17d47391ad1d5e230d3c24509e0f90133c5669d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/4b1c2e2b7e6c2f060f48d372490072e1f6d7b668.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/650a6e518b2ebaad3e20b1099dc80de9babd2eea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/9107ba2d573fc15e9c2dbe6830e7cdec13dfd292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/c4ea9cc3c1b27d207bfb6a9a3e33406b8d358a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866888.20/warc/ee242f54be6096bbe3faff9fd64120b972c6843e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866926.96/warc/2001549770dfd037e71e21e68fb961488995224f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866926.96/warc/4b6d7107cf52983efb68495e211348acaee5fa24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866926.96/warc/84da8bda3d7cc2e632b1e9dcd3ab87b4757047ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866926.96/warc/e95f8ec5b92363c7ed445ebf56228927805fca3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866926.96/warc/f9d5d13c90f3c052e8e34b13404754c6acf416ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/3ce9e15ee9cb4ba0fb599acc9bcc0413a4d9c256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/4abd72d7085957c3da01c8d31a2d034106b48b69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/61a3ae758e46e089b3fdaf8f7d819d18913faf32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/72f16ac1e9d5891972194f3f0ead318f540ff413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/bbed7d9f9215396db4bf5c0519bfb7817ad05fd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866932.69/warc/c6a739b8ea6c301fa52186fc62d917b9815bbed2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866937.79/warc/0238d0fda136ed6b6688ca84b386ce580d151de2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866937.79/warc/1d9f84bd54b4f01e2a5ef28e986d735718fbac73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866937.79/warc/bba9b05d8ea503d4d2ee30138d6b4b15d6d36551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866937.79/warc/deb67f342c64909a22f6276446e9e2a78f54c88d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866937.79/warc/fa428caac542ca512b9966d88d4a545a3a668e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/323663d39905babe3aa4558f869b838fcbf18947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/565713e2bcac1b96a56c4923d699ed0f9d7faedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/903ad1c0d26e4145036968dcbee6732db3ed6d26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/9f474db59d1640ac51b16c968b38a6ae6689f407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/dbe47976919d2c68823d1e2b9ad4e795fa9e91c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866965.84/warc/f2494bd29e5246a6f379c90fca5dc84403b2a127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/1ecff34340c1350ff9c4bc6ae766e67137ed3762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/5d7218cbc1aa8848592fd513445d10495cb45933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/5eb86f0844bf9359320091bd2b14e2268053cc86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/6b2ef829146ec80abd10e9d91a11f297eb9d68b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/7982c0ecf52915e93697020454b3dc1fba0a19e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267866984.71/warc/d53ddccf42c9c838270dbf9e6d227aa1d017dff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867050.73/warc/3a901f2486106b26ef35f15b8f29c23fa3f14140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867050.73/warc/41a1a3a8c98ab02a53641d65958cb5862e041d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867050.73/warc/4ad9de50468240f7c9ffbf829603bff4335fdeaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867050.73/warc/5f82ff8f09d8e67cdff2ded2b7363cfb057b27c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867050.73/warc/ae5e86c47c40433c21f990ddd81f4d030eccf776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/7546cff4cf3b1901354fbdfd6ec227920fee14c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/86dcaa45310ff3a65d5b467e80c30b480b69e642.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/a29fe15d470725da52b2a59769527f20afe84fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/e82ed9cb3f941f23a108972ffd400b7291ec33ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/f4873c682e649df3e8105efda9338b853e1b05fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867055.95/warc/fd3662bff16892933aa6fac5bc8a5402c1b68431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867095.70/warc/3ed3f6ea84011c03bc083cfefc20cec1a232abf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867095.70/warc/40267f9b633e3f87fe044e1581ff50d94670fd9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867095.70/warc/b8b5683a541f4226a3ca2534867f903a5cf64673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867095.70/warc/ba96cabf0a318d06922902a2ec876c07fe22fc8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867095.70/warc/f12cf3d0250dea2c3f677a6b5f03a1f2f1fa455d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/0d06a07996745adba28049cdb913eaa9738f60f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/3f4dbb7188c811a19cdc5bfa5ec33cf2f771ae4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/4c168166ab8ab17d9cfbe07cc0da74e203cc756d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/b8afd7a033cef7880a5e5da745c2c371a0d16133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/bccfe73e219af1871aa4c6fbf5c8937aeb113e5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867304.92/warc/f15c9ffc980fbd4677498abde431b1ce648a44b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/03fb619c0ae979f1ea2c5a39669070255637b538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/18f4e6a945b7e65ca4f0ca5c0238bfa53c5b1c24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/70dc186b3ab3fd3d7da2825821fe189220520495.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/ba7e273537fb32ae424c6711a0d8d1cc986b6b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/cbd1aa3d14d6d640568ad8af192061ceb5a93d6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867364.94/warc/e7ee1b64ca715b78a2ab87a0cacd0ae07441f694.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867424.77/warc/158316e7ad65d20bedf1660f75e559dab1dd5d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867424.77/warc/6553471ec511c5237fa2634a4a002e623d8c35ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867424.77/warc/6cb282a48ddb33c209804e005cc6735f410340cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867424.77/warc/9cdc7769fd9093a31cd51f0c8622d6e1e64d0c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867424.77/warc/e8137dc0b6dacb61dca819b5f32d4f92a89943da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/09b6482c5c2cb68fecb2ba7ece590277b89eb002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/104af4a0b53a016893b4ae665dfc78ab27c1ef57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/7df2050ea5802dd5642401720466eb0b0f6a94ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/9d7f70c44e3fcf049db4f387a8cad69839957d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/b8cb3b12d4691047659986a87a915466b20a8129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867493.99/warc/dac592c8c88f4b7e76520fd9265f58fe98f43dd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867579.80/warc/48bdd43e2eb6bb39a209f473c401ec366a88b66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867579.80/warc/6d3065ff98effdb452bf36c52a521c5cf68299f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867579.80/warc/6e84b51c79c1003b2693c2630452a41a547fce5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867579.80/warc/72350ee2520c1343821df61964ebf1fb0d3f733a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867579.80/warc/7f0d523b280a65998944c878d5718250db583652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/30d406f06aa64389ce41d89beefbf070255a89c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/3df910060974fff1c07d5b6d5e4bbf424c18ee4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/805d03bb839e0fdd99b66dfbfd54d9f576e18818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/b03b8568b7a75a2d6438eb13a85896a9b1d2bbf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/b6fb18b9d8c2b67014f6ff37975827f0a5299f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867644.88/warc/cbabc6942001a1ac541717f1a78f21c863f54ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/461141ef28c903748790c005f148f2f7d9ade84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/5845630e00717316a7c6ba43d216425ca362b8a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/83b04d8402455c9ffafa7316c58b6eff878d7ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/8c7e3034c5f1bc8a35e30f5fb3b70fd46f3c7b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/bbd44d6b4508c6f78056719205ae1fd8ef0e6dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867666.97/warc/cffc0993371e0faeed7449f77404a17a96232122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867885.75/warc/096d65813bd39bde295998beb5b5b5730d74bc77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867885.75/warc/32b32626d037898d773cd512fc6697e71bd132a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867885.75/warc/34b4dc63a62833d627238694adc61b088c6e04f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867885.75/warc/82ac109b6051fb12b5907687eceae57b22b263e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267867885.75/warc/dc3dfbda2a09cb99ea29a44ce6c103178abdef06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/2a93ae0316dc1a7bbf03e48d73ed47ee53efe59d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/3097fd637aa0d777ce1ec682725822cac32c7db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/37bfdbe3b06c88ae8071bdcd6a07426695cc3aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/84e69eb9745704c0a38cb4454ec8050466c901e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/cf438466567059c18572151231f414711dd50806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868135.87/warc/eaec6759cd58cd4b1a11a471cb2dd9e16948bf04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868237.89/warc/2db416fd8c706df988d34e5c623187cd810760ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868237.89/warc/47fab17cea952b8ecf797bdbdc8cd2bc5521b577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868237.89/warc/5a534b634906559109823a10ea3b6cdefb9a23dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868237.89/warc/6beae0ffeda81261ddd2ae497921ad5c2820480d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868237.89/warc/bfbd854210e8d59ad383ae2f9fa2cc2a3f999ee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/24a919cd2e06c68363a9faa5fd0fac1402499b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/42dde5832b86a1ce2567260f9d179c91560f6ff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/44c8b09d1e7d8948111bb17789145271c29b5b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/499553269e0fa1b7ea79167495bbaeba3128d74c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/a1398ef4fdb48369a61893ef976c1632ed537781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/1529267868876.81/warc/c50c6db63c194b275c23ed2c00f845fd84dcbd47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/156b42efdf066676bb1377a311966f3754bd1d4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/15ee433f55a19578a522b738173a9d86f5a28176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/16d286146725835482eae327c35fcbe19c5a9b0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/16e16fddec138d6fb9295eebbadb6b55ac08ff8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/174ebb392f77ddc2f82819a78fea68269144bdd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/20d6b09edca219f92f263d53c2fae59811f530a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/21d244c378cec2982ab6704e27bdfda172ebeda4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/2490ba5ced70b459d9f92c3d051602a55ab8d46f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/26500ee7b1031c56f71a635d501faaafca01407a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/27c5ee40f02e21baffa8cf340ec316bf57334d61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/27edc2685d0f7dbd08bf3d940a9175dd7a443c37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/2a006802f87422e36a6fd8287f90768eafab89e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/37040527cc0a06591961034883943a3eba4a9cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/3b72a1bbfad89bbe905eabf66b70dc81e96c42a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/40acada21ad018bb1396df5e4bc97b74cb194097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/40f2ffae3a511da182da0cfcd6ce16181d75f5df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/4143bfb2b942abc39055d4b4f6fe797010aaa8fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/42507ed1639a5a14a899af7227ba73c957c3923d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/49c63b5c18b987179c0b483c5c018e7d03f78402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/4e12f7b72957f4768a4c6b3e8af227487df81d72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/4e4df696eb18d1fda00a0aae35027f8615deb150.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/52d2571fa02ed850361312ea7af32c04ddbcf796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/562014ad2c8d248e4704b0b29395fb9b51d7ffd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/588f518e2395aca5679df1e9cd7612555a5326cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/59b7bd63e4883cf2946615a98993a45c5c079fb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/5cb4fabcd65c98923257b187cb865106f2caee99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/5eaf4b75671dadde8118369717835192f7de2f26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/60b2a2d31754ffd7dc771aa8f4d169e003e8de5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/63b80ca7b367cf8245430f817fcf872796b854fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/67febb409b36a5a457192a8f5045f3b8caaaeba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/697be789339bdee78f992c113268616317aa628d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/6a24ce8e74c2b6d42a76cb585744264a8c3c32ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/75ccd87c1515716c4091d71529e1f81be198eae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/763036f1ea0bb552a62289ca33b8383ccf8a246b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/76b094d857a5a822c0b788a23c03043709cfec07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/80d70ac840ed1d7480acbfcc45dde5dca8b875a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/86256a4f89e5927629932b20c6cdb212b66710ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/877fe730091da8b9c06568604ea2cad04f29d85e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/8b4b7899e2cabf7f61237fe4618f0caf302a691f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/8b89ffefbe31df8780d9e8dfe2fddde95b865729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/8e9521c105029c7174703890904922b4225ce285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/8f9212fb0f3d1151c791013498c5ee13daf0f158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/94e2483bff4c76e566e99d10860fe3dda7c4d742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/97eab51950d38e35ce7c098a19952e0aa2ee5c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/995867d80d73d9971a5c8ad42f44990de36c0ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/995f2970f54397510c85f9bcc21960e06433b2c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/9972fc7139643b5f46d9bff3a99ebadf1adb13df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/9c04e9b7d6b129b724d9d4f75fecc44e09d3dfa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/9c27bcc96568d62a60fce18760632cfff253d478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/a0030e487c9d84bc3d37d99ad5d6d03da90fab67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/a8688608818db4517f4d32fa284f30838079279c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/aad964be83176589389e899b97deba7b72c8683b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/acd9391c2372b9c8c2d2f304e25de29c37a04952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ae02592b68f5bb631745422a8cbba9198b6f65e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ae63d5eeef39ddf6a7335181e00fe5a0ba8aadc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/b768820b19c71f4c08accf0d69577cc79f6d7f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ba717447a8a6003d241c73d7e88d4bfc47e3640e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/c4d6b5d9e8bc6d6ffbdb709f049e4c2ea01ce5cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/c60959d6799f9f11348eec1ddbe7d35a01b248c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/c8de42f641e73b223d512bca8414ad7a94c67d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/cdadc9a7ca6eb7d535c71cab328e1134a2c7a562.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/cdda870787dfc73812b72d9facb8c73afbd489a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ce15252dd1d3411863c8ccfd05ff4271a2831750.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/db842d4bf5cace03275be13f840587b918c04de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ddd67bd631194949bac4f594c2ab10660b1e308e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/de8016cd0adb8253a6e9f246a580f472710aee5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/e18619945d4c3b3ef5cf008c88e41c7db311842e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/e1c745f7b102c60c53366a50da826c6e6e136b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/e6339a94dc8af0fdbb8de6fae4e8aa891754ab27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/e9a1b81ea2c9e45c202b05f59f513306e33a06a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/ec83055d143f3b2296d8fb3b27ad70e6abbe7659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/f7b19284e0529826dbb2dd79334a43577e8583dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/f8f6c673ad6a542cde62a6efcf091926aa303c3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/f8fca67b04e1e47a4efe94f81cbf3480f4c837b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/fa3c62a0875223ca310a5368971c4e227031f4c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/fa442e58cb2ded3f25ebab548ec215871f0f1046.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-26 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2018-26/mix/input" + output: "/tmp/cccc-CC-MAIN-2018-26/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2018-30.yaml b/configs/cccc/mixer/CC-MAIN-2018-30.yaml new file mode 100644 index 00000000..4535acea --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2018-30.yaml @@ -0,0 +1,754 @@ + +streams: + - name: cccc-CC-MAIN-2018-30 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/03eb0847729817afec3dc7dc0bb7937d55979167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/064900654ef0eeec943f8601c0b261e3517be228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/0825e5867f16994f19182586039ab98e2590b576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/097ae82800dfba8805058c118511824f78f1eaca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/09d45a7c78e1fc228d80d685319c24d65c995762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/0fa5594c4293f0154af250794580bbceffc94b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1125d3039f2604e41bcb12635ada71fefd9849f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1162369243fc4dd0f09f18781941baa4aa6e4a06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/00a30b91e3af498762c5fc967f49471483263954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/155c9f961d5a50299a501438344d058d15dcfdc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/605feb9123e482332bb4961112b0c89618949eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/8279e125308d5b49b11f53bff880ae7c400e0c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/c7d7ed9bebb0708267692086474b17bc4f80670a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588961.14/warc/ee5b832778adbefd31938fc24c28b52031f235cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588972.37/warc/09585c4e642deb318018b7e5f89e0ac339c0d96c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588972.37/warc/3dbf524ab7ba931441eb5028c8ad1a21d8068343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588972.37/warc/a2f73ea2edf8678d203604f152ba895a4941d637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588972.37/warc/d276b11f2cedbeb67a8bfe0bd987480096e4f6b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676588972.37/warc/ee931076571f51eeb58fe67998a704a0ef64d297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/0772b3a56f442e376b485d73549e83dc0047a36e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/4ce3eddc5a4ad4ec7991c7b58cbc0d6d5a3dc0dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/648b9563119819293f9c6ec3a7b976aae53c5e19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/93f6e12640554a8a19589687d096b3dbaac3e525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/95bbea578b5ef35624e58c96477b2278c42bab4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589022.38/warc/c385be93318e5352f18e6d076ed3502b7c93f66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589029.26/warc/0f13ffea70d91173b99c7350b503f6ca355396c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589029.26/warc/460e0bb4eff96f6c1ed99c7fad9a8a75a1bf68f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589029.26/warc/db721a6ca2a71fc76c310d81170a273b7fc7ff4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589029.26/warc/eef542e7b7769b5d0219c10c9998832993ea83b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589029.26/warc/f70c485612f948b4b4e278a95fd844cf79196440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/09a897f66d79035a95f6d482a1dc1f21f46a9e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/2df94e197dd34f4d38ff08aee5130e2171310763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/87f93236fef9b7099af70dedd25d6b6641700d38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/9f7a38ae05d4e8d6dad883b6228a7ab55c6a1411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/b55ec56a6787b3ae408e56f3f1a12eb412a26ef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589172.41/warc/f36711e07a455568499c6becdda307db917eee3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/0a42fa18e8a3685096743b8f1b9d15570845ba48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/121117fa4aff708d2d7a52d7649d6a741d95109d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/1adf5952088f03e04071542df5cc7e5a8cc95108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/59b24bbf92a12f5f7993976a1c938744a5e1d2db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/6eb807090ffae5218cf4a22c32705c225455de80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589179.32/warc/9c8c0508a55e0269931c9d3dc4e9f39bd26e11b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589222.18/warc/2564552a145faceab7fe7d80601882c7508f3faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589222.18/warc/2f2692caa0d2a89d1f7de27ed9c181f314dcdb4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589222.18/warc/5b65827d4864e5c34985dfbf66c4e56e80ae2730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589222.18/warc/62edbee8e6daca1f36c3b7007b7485a2ce8d624e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589222.18/warc/69f4be65d0d9d706301c5cd789e80cc2962c4f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/2b63e84e95b24a64ddcf369ef488f30f642ac52f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/3a28fb6bb4380883040724c557f6faa8cd559829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/4daae7884696ccf0e51165f3577b4ae39c3ea2cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/528278f5ecb4a0b89fd507e130c4cfa860e74461.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/5aaf9ec43a5a31600d064d1766a6bd4e9476404e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589237.16/warc/aac44c2a9050bcc565bd025d2bffc6ee243c8698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589251.7/warc/345f8aac6263baf28eb07569500ae2d21fa97bca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589251.7/warc/7acffc803f9608718bd874e5086df99f3139f557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589251.7/warc/88c1e570b6fd37236863943835ce2c06047ad61d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589251.7/warc/a0a062b8a2e08b9d5cd25b618fb18335efa41681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589251.7/warc/b9cdb8b8d07e1811bc3a2e4b258cb8d03d74af7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/237200afce776f595c9bdb9c23e9405de0cde193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/32903e8552f8f82187685a5bb60926f8a985784f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/4e277087393df5a5345579bb4ab31fc0c57049f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/95965cc6a46f8551d0193fdc56c9df771cbd7bf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/f4b2c57f77c2e61d7c2834240f51a81c848e18ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589270.3/warc/febe4fbc34f21033f82ed82fc3ecc8098a95f130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/0a7ed0a824e3bd14931efa3d08c617da785de9f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/6d90b5c52efa0890f8a80f7522a7f1dd2f12ef69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/8ef6c02a4605918a8f5b00f04291156d8f12a9bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/ad28cc2c1e3e4c83f44582c49b1ac16854c0b41f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/b4e5f2c70790ffd540d484bd9103ac7a3832711f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589350.19/warc/f2e02f91816e76d6c5f991edcde96dcf8dcad412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589404.50/warc/050f1df2a4b9b6cb0262e6c029b3128ba6261c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589404.50/warc/0e3053d109994f030a17178c913f2f7c42156ddb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589404.50/warc/500f891b998584625136c163d20135ed32cba26a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589404.50/warc/9040a8f7f995dcbf2e4f04da2ab5f3e9bd89b45f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589404.50/warc/b19cff10748530faa9a53bbe5f7aebc3739cc56a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/16138c212cac82bc8bfbd39c23c98201d329e789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/415eb19c50e43f8434107179e5db7555693a0a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/63201d95edddde65625f3ba71f1dee73f2c7b291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/809718cf8b7fe72b1f571f48a9425a820c77c571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/9c01c82e9c51b51becb20bd8cfa8c8dbf75271a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589417.43/warc/d78e16085bdab40b4ec03c8862d67046538db19e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589455.35/warc/1f45a555335c1cb5e3c6e39865ed797b8ed3c0f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589455.35/warc/219f39856c297a218447a07a1077db192692bdc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589455.35/warc/41bb6c33a2400bc5b69be8d2316c2c9a9723e349.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589455.35/warc/9f04138979df2f8d3454e9ca8e4072c083f639b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589455.35/warc/b4052c2009e65ddc9a6dc2024ab0f65a10a4a44d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/0eba424b9548f8dbdb57697d6720309eb499eab5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/1ac003648058ddd82f866354cc61aa8513ae8f31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/36843d0db016e49356d565f3d56e938c9ca9087c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/77a0821772b5d9b9da7e167ea9e36e85572d5914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/b224b705fd043424e2a00703d35d53c707eef2ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589470.9/warc/e17eb1559e5e6cf3157e91e15903ee5f30e2eb9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/1742c240713d213fdbee9e043c58feffd976e5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/2a38d085f81d167146ea1c845a42f1aa147932cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/9980d328ce6c857245c405e2a85ebd9b05d65f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/d1a19f7f4d1442eb4ff38f753fdea54c5459506b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/db403e888c5942d2df38437cc1349a3b0cfcde36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589536.40/warc/f2c9ff2884fe349bac50595c341368b1cf22bc22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589537.21/warc/032e6c3332666b53586f221c7c7cc159b5e09a8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589537.21/warc/761ca8996058abc8f631f671894cf54730da21f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589537.21/warc/aed36ee46d2ecf82b8785595a842919b28d73532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589537.21/warc/cfff23ebb7acbf2269953e8c37b58cfcca793be2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589537.21/warc/fa700d0f486e33d81b5eec04e9a8cba27db95e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/7728c087bb2ac5de63277af355a82a657a2e7c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/9901d3305e7f8b8cc3b7d398030fc02dae0c8549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/ae66a9d347b6f92cb9a572ef60134acc026af3d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/c712fc82c570cd737bdf8c012e12bf827fc5b961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/effe1e0fa7af95692713b6e5a4dc03e9aed4ce64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589557.39/warc/f1a1e64e2d9a469ab7cc2b9ec34b9dc2aa4d8068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589573.27/warc/178865dc5daf318f11aa9fca8e4acdb2e066abbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589573.27/warc/4157afaa7a431169e130c4a1af0a05067695475c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589573.27/warc/4cc367622f7c0c13a2c091cb4e31814ac02642c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589573.27/warc/db9f9abd8fd3e1fa9fb1cddd28eb46cecf218ded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589573.27/warc/df919f5f8ffcda998d9f13892f24ef9d792bd1ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/469af8c208c3dc2bbc72787285957ac429433cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/55e99ac5ab92b33948b6bb8f5ee59321b8f21395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/67023e466d1033acdeea61e97dd805ad0beabc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/89a8700e04ade014ed662f4af5c80c8a5be5e890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/e0c51384acf32fd62ee93f59c70b0bc92550f8c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589618.52/warc/ef8daa77577a45fe355662d4f92e719b6ffd01c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/001001c3f4a69dd17a8a514b3002abadb7036e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/46a424dbfb1534dd9a6bd7288ecfedc3b282dfb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/82a4b9e20bec12fb22077cb353ff3a14e168e64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/892a5df6835bf1f32fd4ad13d879fa4caf8a441f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/aa3401bc52f7cc5b6c3ba7bb415035fcdb727d93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589634.23/warc/b642c80e85a432cf609c518af6b26b80be43f7f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589710.17/warc/340ae28648f4fcf27383cc800e812290e565843e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589710.17/warc/4414d93688f5f720872437401d3c5a7981961923.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589710.17/warc/8effcb91a5404b11d713fdeaf93a01723a1bacf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589710.17/warc/b6d028f8f82981b7883eb57d84b77f16fb9f420d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589710.17/warc/bfbf99f07db111cb6670d730ce17d9387bf8ddb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/0375ad910174c6c22f44b7d2ae1584120f30c9e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/6b3c1c1a3454140764009a4859f2050176e5cabf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/b467721cb8e751787b9a9b3e11b8836c2257e84a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/cce7587823eeec1631d5c7bfa2c3a82637f53801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/e9c1404d084e5361aa29457a4b8318265657908a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589726.60/warc/ff50b834adbc2e5c9073277ec883333a8df036fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589752.56/warc/78cce0221b69592fa8307087ecad892c162f33cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589752.56/warc/7aee02c193d15e89c446cf93582329612c34331b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589752.56/warc/b8e032f05269d52f6462adccc2b49f510641e3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589752.56/warc/df5c53b160ddc36cbc66d69c962a77c485dce50a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589752.56/warc/fca6c886629aea70641230a556817614cb40b903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/03bfb36d72c65c10580ba0417135dee2f98265e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/12fda16862f9979f8a4bd7e09ceb7b11103080b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/2d1c078cbddc3ed1e3cb8ab03595283d16a3fb3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/52f9f28fedb246ea7015a3da7274218db2c117a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/dcaaf531950fdc0e0fe672a3907f4b2b542c6915.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589757.30/warc/f1eb1696c589a395e13f86bc7ca5f11577eae60e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/0301f38fe39f04de1e708cc9f94a2f56d3a9b158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/41e64f4bcc489fe8dab67077edcc99e00ff6c366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/4b8c71596df1c4078f63e614311f706efd5bb7bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/8d0ed0559342023a13354ed7b1f5ebe801a7f0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/b48bd245cb13e181e3f68ed71aa98dfd025d4b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589892.87/warc/ff06845b551e2205c5e728592c39ac9bbe425ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589902.8/warc/1cea8d5dc9b4fd91b26afe3e173f447b1fe3b13c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589902.8/warc/3db45d28d40fbaf207edee58a09faa625430eb50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589902.8/warc/7d72743066e86ed718f7e776da874f07a01a67fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589902.8/warc/cb421a613c44be706a31c149e30056ee7aaaf65e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589902.8/warc/d901d947b6a19be2d15bd3117ef6f5a647901c8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/0d810e6daaaf6b2fc1ec0bb647b3111365d2e3aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/0f4109d6262e33f0b51e4758803f2f60f8af3afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/66e642cafbc3ee15426895067889dea769a4893f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/79ae8e0f126e410e67f0edabfa203d8e55ee9046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/7e2d7521c588f261802784412a5447fcb386ce67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589932.22/warc/dd7a2dacabe721cce5c2fa11dbea02e522e8cdc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589980.6/warc/1e0bd4a4d4850062f925aa6c5335830bb38a75f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589980.6/warc/28dd7b37fd9e6a64a219831fde47b7de87fc5b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589980.6/warc/3d81ada008ffec1beaee123ebe1474bda09fe675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589980.6/warc/a436a418caba6df607bb11c35d61d1e46dc0f76a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676589980.6/warc/c90332f71fb0dbf665f0e93524b9079ff5914a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/102a7ea62eb1e64aa5ededebd2239bbe193bb155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/6cbf281ecca2bfdea03613f95543b82c11376db2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/99e433e0c42e680bcdb78ac90e97ba67f81ac15c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/a5407683d83bcdc15430716d64f2d624cd0e0e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/e95ac173a7f55f8261bc80e926b0ec503545c87b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590046.11/warc/f205aff7660391cbad17ddbf91728a8f4e4fe0c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/23ebdc7481c99e190b2997346f2826d7359c6797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/432cfc77f17bd83e45c997af2125b228f6488c22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/928290657b47df87182156e777ca10cd07b6cbce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/b7c741829833a794b614bc9b88363491c92e58da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/d7f33e1fa6a4d9842dbfe85d37b0ea4d08e2be00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590051.20/warc/ee4bdaaa30db1d533c7c23fca7a0bad106373d96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590069.15/warc/594875929f87414403f5c10e17a781e6d701472d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590069.15/warc/8c6591a80afacd6c803e2f344894fc574d37ccec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590069.15/warc/a51d06ae67388f2ecc6d3512e3ae3ad63b3b930f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590069.15/warc/b4c95438160e5574eed26c1f01a252818942e2d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590069.15/warc/eccff9236234ae841818e7b6bf87fb62a776cdac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/26185a82fd4e8a5266c5aaf55f43799598066e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/9135d9a590b0a4a22bfddfdc507e200743631335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/a6d2f905fb48eaff64c7f493459eeae57479de2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/aec184756e8073c70da12c10a62505e2b238ffd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/d699127b397539a29a8cfe2782c82916fbf434ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590074.12/warc/d713c49f1ab0dd7346073eddb0f95bf1ba9345ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590127.2/warc/6466d617d736ba7bfb669b51c95c1a3f64038714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590127.2/warc/6542874da4b896ab1ec5fa0a2dab447b98f29038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590127.2/warc/6cf614240b9a937363d02901595ad5cf91a00e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590127.2/warc/9dd5ee237103414efd9da8c6ad7b8a2b619da96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590127.2/warc/a634ad7fab5bd2097869f51c803bae4d1d713f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/3cc236bef56559d6170a0a84cd4c4e143dacc967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/6555930ec06d4c6fa11c2cddf35b279277bfdd84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/6974a01959528c7971d8a1a7362f0783d0c5a11e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/87b5e7e115cbf8ebb910fd9887274862e47e17e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/9ac66153e318ea7da7dd710043991654231d2b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590169.5/warc/d520f2f683b719a84f98bc8bb23436969584b705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/03c398b44bc0d533fd89966949d1671bb76131ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/238e977ad5dd76e8f21913be2031c3e4075189bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/24c50f20a48179f28515767cfdf9852bf0aa3fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/6c5828d4936cd088a0cd5d747a6d1a8f45dcff51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/a84142628a8015ed18af74914c10226b10c59860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590199.42/warc/fa1ab486067f90da20d409675d641dd3678dd767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590295.61/warc/1140babd16aa129e344ab4f3d4c8c981730b5576.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590295.61/warc/17232d1538a1863e9919338a6051c252619dd804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590295.61/warc/3ad9f05673a33204ce20594421507134cff18959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590295.61/warc/a34b632d5ce52c37d38305b0468c9b5d116dc676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590295.61/warc/fde85ff18f02dd7f16442d2497af26b2919d636a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/1964cddc90c9a8e495ef7d94813983c2648cbfec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/38f2665cb96e6767adaa3e53bd51b6f87d012b84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/a6c33c3f25167e4123df569b4e78ab74a675c91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/afad1f52d6173680fc90798a1bf6d411967c558a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/b5654b22fd86ac2ac71b9a29db4cf4fa1ef6014b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590314.29/warc/d1a9a0ef52c6f1eb6789e5054f15e729d7732c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.25/warc/28b79b84cc15ac22e25f33c990eb146ec72ddf07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.25/warc/53d003f05d8ac13a9cdf8ca7cc4361c4b9c2dda5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.25/warc/703c11cee4b26eec699ccd2b116cd0985e390fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.25/warc/89db85c9f3b63f10c51b7b8079694c91118ea36f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.25/warc/cfbaebc335447a7f8659768284b37cc8aaee68cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/07014490c0329c00230eed58ac77cda2ee737140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/274aa94028f58f2bf4537599831ba3a6429dbd06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/72e242f0ac3a6c614a95a26cc69bb30ff51788a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/9ad44651f2335f4dc043bfd7abbd41cd29ec3005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/9f34189354e5b11e368c182e5d0262de93d9c292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590329.62/warc/d55bb61d6ec4fea8e062d6c5e557c7e04722e4f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/52f0a3b9f7853b13937eab3a29e6a4affe03ff6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/90e0e953373a64fc1a47edd4db82349fffad6068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/b2ee6d2ef7d827f71a6a45a6f887a191a67a3dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/c419ee6495b4b2a37349a820878084f55a249765.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/c74dc65b7dee98e819799b9c6e6cf2f10cb3f167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590362.13/warc/fa7eebade50c2943cdd0e6bd6989b2702f9f2a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590443.0/warc/34b0ae5bed591819b2dbc3642b7f3f42ec03c247.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590443.0/warc/8a2b83e56d5d4f3524ca4e7d61bb5f4e1cedcac9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590443.0/warc/9965b83eeb5960adc3d76849ab804c51012e3d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590443.0/warc/a8c39cf6b859f58dc26f44d1c2b89b7f829a76b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590443.0/warc/a991e1b5ec836304999c5599b1420d4ff61b2277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/0086cb57b38c34b631aacf14cac0142ce7070afa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/8501a328a20d93148c1b0a5c27d377660db7fcef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/8daa5a2fc05b50386423e7730ef1d7d0e3a1ded7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/9944ffd633b27d91ea2ca1041c2e0ca0dc1e0581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/cf22ccdfaf680f1aab416428c7c9fbee8402e085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590493.28/warc/d7f839a86ae68673fa26962cbb121d588bc7b84f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590559.95/warc/12e6a73fdccb9c11bb0e1e29b0aa00554b9dd5d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590559.95/warc/240a8a0b96f7a561dfbb1eaa01499a0a7241c2af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590559.95/warc/35dc143fb3c39995e25ddfb6763bf4adae7c4a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590559.95/warc/866d8d9a0407ed412efa66747b83ebd7de361b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590559.95/warc/be83734f403d32fac15144315d485147b22ae68e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/21896773d5b7179e08c2ec1eb09d5582468860e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/5b048baf5d21c1fe06ad023d62eb74b5042716d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/899572d949b530b11bd821098233254ab0a333ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/a0a603e63ce15c1e05ad41e23d434d208de7cba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/c136e291aab78d2cdaea1e2233eeef4532c5d087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590711.36/warc/ca7be64018dbe3ff8105902144c093c60d04be9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/0da5d298d87cd1df20f04fa7119835c3ffc9ae07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/7d31a6076f49eb31c2ff3485b4eb8f56f660ccee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/88b0c5c8209bb34618e6b44d13eeb661d7643514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/965cf522088c786971334861dc511834dccf41ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/c4b538fcfe8ed71be2d950a9c1431520c9d36dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590794.69/warc/f1885cd2938baa53745a454203ce0726d7f76257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590866.65/warc/6210513972d62d3a714f0ae94f599c610d20cced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590866.65/warc/7e208b7627c48772f11b36421131c72925959d95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590866.65/warc/8eba5b5ba132dde94649ca4090e3dbc90f15e917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590866.65/warc/a777d98e3c62412b082dd6a101b8c74fcd0e9c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590866.65/warc/c58b79007eab9fe8b7665e7b9a122cc8e2471db8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/09f0d8f4b03e0a8aa3ef0e38ecd4ff5874e94e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/215ce9a75d2aa23120fec0a0e11e4ec9acf5794d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/27e1f4dfccde47cfa09f89de501fc5378554886b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/7be8e43b87665088ab0b12d26f24bc3ba3d833b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/af424da316f58c58ff36c49805c56837542bdb2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676590901.10/warc/f78865ae8e4ab57bd0cb3c51dcee53aedfe7da60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591140.45/warc/17eac5278b319a41970edd2af984ae6cbeefd13a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591140.45/warc/52edaf2a2220127ee59348931d57119a4699fec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591140.45/warc/5777215a242a46e83e4dfc80cabf52d4f82098c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591140.45/warc/8b69b6155aa14a439f3e2d33c5017c67bc5ba72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591140.45/warc/d93a4ec063bba95e7cc83e8ecc1536c87f3b7777.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/2acfe486afd49d1d628523f73183ef6d295c91c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/4252f677f68de7ac3df706b41c0e59ab334b2a16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/5be3a145083a1bbcce14f9b38efc3a0a5cd05856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/82b9ef61201ba08d878a3bf8222deccb3eeacd01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/942ce08f4370dbfeecb12cb8c9dd0833e6662ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591150.71/warc/9b58dd0f595c4977b0532e5cdbc4251a04a4c985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/09cd55e7ce63370178df90085ef8518479df58b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/674b333268bbd9489dc36d02677e34b3627796b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/aa3027da0859e200c424342c5323ca629d1aef9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/d18956d49df7c089602bbda0b417af17082f61e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/dcba6c3e6c2847736972b92a1920fc8db0d60a4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591216.51/warc/e1f52395fd1b7a7e528fdf03d5bf37e638f98dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591296.46/warc/16b02844ac4ef1b70e8f4fa18ddf9f5e3f1900ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591296.46/warc/35534b7738b37b7fa32be6d519bf06a42e3a48e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591296.46/warc/5dc31146cb57a589e7e4e22922791f483d08cf9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591296.46/warc/865a5d5bb09403edbb67c14c9c4f759b2ea76554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591296.46/warc/952c678a1f32aed9d54f523f25351871f81ea262.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/0986edbecf3ada9329cd50d1fef02f23234ca84f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/2a3527cabbba0bf4f41dd8305383e737fbf41a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/60bf50e134c9fcf24f5a03eed398e7718e156752.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/797de6fa3824183061539c83db33799ab0a85436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/bc55320bce04d68d202a945e0766e365b90cd974.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591332.73/warc/d3ff21b7c787510969444a06e74e9e8e43d20ada.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591455.76/warc/0dc248f9ae9d071e1c818f3fda486cb1de0a1aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591455.76/warc/368581b4249bcd78cf2eef6d88587d2373019a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591455.76/warc/3d054fde5fbe9ab26a378a998a6d724eca4afe34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591455.76/warc/be05273196bf605438a8a3a1668a711f04a8513f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591455.76/warc/cf69479a9eb545c2aa29b23cfbfff4f6c9f36772.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/039b86344bb41a3fb6cdca1f2041c29b473fea5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/493720df0656880c10c9ca369862eb8f71f3b0e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/b63bd7723ac4e6a7b94c8a4161589bbf1637c348.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/c7c9620640005dbbf44ba4875370e80291359bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/d22f53c55931548f663ed1fd99264aefd7633aff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591481.75/warc/e103f1e6131a5a3bdd4f179872eb8269098efd67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/046607b0c420d8bfeca2de5024a7a87404fb44c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/0fca09c6f8fb750210ad245aef3096253047f7ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/2036cca11332ad8e04bb00ca53373d79abe0ea97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/3b7678df6679d3603273448479e480a9882bebe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/60e5e5768e25efe13e848c23a22866726f465462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591497.58/warc/deceb04882bda6d95b378a225e87abacc67c7880.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591543.63/warc/1a93873927702b4e4d11137017f3003a286559e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591543.63/warc/2aca71acfbdd3b6b22caa4c4c62fc71d355a6297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591543.63/warc/49f5e7a5a39b0ba01c750d514b1f741995522e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591543.63/warc/b1fc32f47ed36d68209a464c18c4a36841cc2856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591543.63/warc/c7ae4914db1a1c115498b09c5dd5487d36d76926.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/431a73bf4d6fc578d015e0bbaea9b6623e536661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/468641ecb5dd87d825c0645c4cb33515b4a2d66c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/9f73f4e96bd6ae0fedee3db7dab4be3c8635f995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/c1e8febb2f9d3da1d5276446d5982d8956c08ed2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/de02f69166e92ae20f1f84a2d8ecbe3bd7df94c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591575.49/warc/eb19a4853e5b396d4cd7bfa6c40aa1122a095b04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591578.1/warc/09ada2ad43ecce218ca276ceca083bb305b0b8ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591578.1/warc/40e33288be9a02d5522b0bacfddfd96b741b173e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591578.1/warc/aebcfd0b02973c106f47ad58f3cecce51880527c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591578.1/warc/ca8fecb424d6f5f1474323fa640c533a6b3008c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591578.1/warc/fcdac1da31fa41b2456cb66307c4e0f66a1061e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/71456f589686a65e5557336f453441612a8ccbff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/84915cba93d862013b8ef5892aede1152087dacf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/929fb372b3670a7276923d14cba8974e4eb76f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/d1a0c7112409a75fcda2d3fed2560cc0015b1548.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/d1c5afba1a5d4f1baece8db865bbfac0526e0aac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591596.64/warc/dbbd3a1edebb61e0c3e24ac8bc8c329f46bf3bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/02450d6388562ba17544300967a33d3850cadcdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/233f52e3da446869373dde1989e9aa7837722835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/2f9b4259a2e845134c10f7fa9661360bb9ee29ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/43a6dc59a92871aac4a93081580876bfa3fce74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/be3dc7ff2f1f5925078dfd4e47e72d6c463d05e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591683.78/warc/f23250ad3fd4bf64721c279f4b18e63a10f5d77c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591718.31/warc/2999dbed708e74f9abbd3d538acb94a7da2de295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591718.31/warc/472dbb074aa8dbeaebf6940d393d75fbb9eb4bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591718.31/warc/52044f65b823fdfc4067993f2378eca291a9d58b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591718.31/warc/8502a9cec1555bb403390da14935ef5922e2ea9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591718.31/warc/f3099193b20a0faaf189264f32cf80e06ede9cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/3fc538c859477fa74e20afbc31af4c04f6cdc116.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/4b8476310ccd62bd121b1c5c192824feffd685d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/57b77c0bb82adf1d1b3b26c39c01b91028473671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/6af6c60b87b54fe36faf471d7a79d0bd0d6c1054.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/72ce09d4c50b1671cfa7fc2a3ec92fcd1eda80d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591719.4/warc/cd60d10dea33f1bd19eaf68418366e882c2754d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591831.57/warc/7de38280c426323cdd0cc636316c21967c21c9b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591831.57/warc/ad0023bf37f79db8a3efa79d3ec1b3b673a0768d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591831.57/warc/af4f2148e4446a3d1316db65564327555fa18319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591831.57/warc/b317d648fe004b0da946e3acbf2c72d6b077a3d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591831.57/warc/c76e266950853d87da731f2b1ab24081fdef5fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/321235bb43bf18e377753841e2d97283a3875df8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/b9ae718d55ff1c25ba0f8421f676e6ba5e972ebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/bffb495349a09fe6b37b05c2c749a31bad710743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/cf251d4129dd7c0974255b1ff8037377dc4fb757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/d1175a10f7f08a779c48b5bdb3476feefaf5a913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676591837.34/warc/dc48d46b207970c2ad13d35202f38ee9c571a624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/0a36c35ff926dae93cc4fb66fa90c24141142700.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/4c940803a3a0a0424170933543b1b41d74fc1fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/637aaca2f148d5d9c351d13ffd7d654799f06304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/641eda63c6823592504d4dc3d1e546fd3610cc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/68c8c1923087e88e70edafdb055b5054909c930f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592001.81/warc/a067acad316805a5050149aecd3469442fa61669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592150.47/warc/4cb29e615c7330f4dd7b39c86ab5597fa9e532b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592150.47/warc/64ffc4c90db234bbcbb19ff051d56365b85b5061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592150.47/warc/98a386b3b9d2c8cf84a97fff5c05867f0f215840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592150.47/warc/a366bb64ea922649feeb69d85156714c922fc46c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592150.47/warc/de3647561c13eee848b7ab2635e1735216000bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/1b65df9827fabe150193ecce4b385a58cd9a1c21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/2309605cc78dcc919629c153ffebab669b978fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/32c4f97d5787e1bb7073c68fb82af0ee9e11e2d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/34605215a7d7708046232a9503384918f608a357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/5e958371c447e7516b1cb078f900dca32d25eab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592309.94/warc/f44b115a7be111921e82c4de7034882fed7ac2c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592387.80/warc/1e0406b2b571acafc062876dab22dc479ec1cbf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592387.80/warc/87a64e313943149c783649172715de27f6be84ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592387.80/warc/a5d861afb93f51211ca122087797a9e5d9daff9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592387.80/warc/cb78d3165641d687259dd286bef0734517a6f934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592387.80/warc/e5fa990dbc240365e6f6437526369d9932d8343e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/56525b726960fa85afc462f83dec002538cd7833.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/88902591b882117f4cb272b49a14ce14f64cb5e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/8f78f39cd4c92457437cb5febfe8b27911469bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/ca6c88a7ed439a6c9605312a2a73e683ca2eb2c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/cbbe8ecc87aea23faf7034576fad27a9a36cfeb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592420.72/warc/da1b97f18811c880461a9c08a1cbbcd634b7b9c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/0cada9072ff8934495231d1961985c2a537f2917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/4353969934511ad344882cbf2767f3236ec15276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/63cf0ca523d8839dd91602634ebf52a22939a7db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/749bffec006952550e9d18789e62cfeda85355cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/ab159bf6dc90a7a940d97efa9edd896295d818f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592475.84/warc/ed78c9d54e9415a4f04347008271558124ba05c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592523.89/warc/33d833abdda8ba5efa3ac9a08a7794ceb2d03ccd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592523.89/warc/6c68e70fefa092b44161d8398b1a0b0197eb44a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592523.89/warc/81fdb28dd3bbd2fd71a1b0a888b7b353314148ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592523.89/warc/a74be61b9bbd5558fafe76294bb7a3671961f3de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592523.89/warc/d3084ef73cf1eb310a55b07853f53335b0025f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/208edd4a571ccbafaade43b7f26df923dc249047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/31cb4aad727c3ea348fa06d14e2610d4aa925590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/41220e57d476d7ff4038694970690fd0701b7e0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/7c9a86a067c0bd6490c0d49bb761f3c0c6fe02e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/804a189f79baf40ad0d62e115596118f8d7d613d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592579.77/warc/86a3bf3ff58a2013cb64e37f50942c4736404001.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592636.68/warc/3f4b25b4fdd3e50c59cdb1dcc0b9cb5db3d8d65e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592636.68/warc/4cba89c258b0d442f993c08e894ebd73fb3700b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592636.68/warc/6de9e9c9e8c88bbd5e390479c12e323749c9d584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592636.68/warc/ac3c44dfcdc9825d34e4b3bddf43500f12085771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592636.68/warc/e898ac23cf43cfba86bcbed44e84969c7d328640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/33fdabf76966a6686d28554fa764e381861a890b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/57aeecd32549ff6395cd779b2b51e401ce91a9c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/7b89c462b5e2e478f3fa528963b77229e92a6390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/7cbea2f8264984575b0db3567bd5bb67ea27595b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/a9973deb7a60c7a7bb6bf62036baa347de47e4f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592650.53/warc/c59daa6766b8691c70cdc99dfe1c815af20064d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/1720bd3481c19435d0492a37c3edae8119ce7ffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/46e8d417f45d4bcaca2c74439f41a764ece560e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/a614d5421a0cbbeb7032fbaab1ac9136f392fd45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/aa003b76876898cc5c1c96791855995ca769ec27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/c433f4f2170afe7109e57d632e81d8604573da9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592654.99/warc/f78187f6a73ed4c8984cdc2a46da478b02ec311a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592778.82/warc/109a11d7e971365df7edd4e8d216d5049fd816e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592778.82/warc/485c5a221a700ca38cceb8826bf71b094ca5d1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592778.82/warc/54357ae426b0cabbb603846960dbb66380798f49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592778.82/warc/c2c4fdeb92f153cf03c439ed129bde5216227e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592778.82/warc/ddd5b045a138caa76ac6f52926dbc8917f97002e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/0863852c0639fd88e947c2e632849cccf3ed54c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/12f7905de138d4b2a7c4e7392b153d2ab0f8125c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/4f1b6cb611c602b1115447acba4cf4d340bb95ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/577fee7f3f89616e17ffd6811467bffe42779c62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/72927ffbcdf746b56bb89a0b3bdd35eba2d8da6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592861.86/warc/f250fc7ff30bcb402fdaf4dbf1fe55c04b645498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592875.98/warc/0103caca1a9542984044c79d551d36945b6ce2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592875.98/warc/5d789182fc8baa5438346a5e545f40b4e6070255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592875.98/warc/62d460c9105e319da647eaf3198b02170243c743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592875.98/warc/9dcb4b9a790b5393ef8c91d592a12d71fd4c34f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676592875.98/warc/e5a8f821780e1b17a5b35bb7e512c2ee0e11ff08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/109b6a6314de05db6267925781295a50665ef456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/5137202b4d6a662da142433347b4b1cb45201819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/72eb739f16d52e73ef6a456f6d5c03df105d9962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/a2e44d7f5124ff41fea140c1c926cd580c4c9afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/b0bb04099f24e963d9418402578ea788d45ed4e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593004.92/warc/f094df6a9c5806b64f12c12fe7dbf6ce598b5552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/0578bf7f6adce499609df2d65697ddad2e79b351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/2b44609feb52588c6f2f104fe9f41779b24245e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/33fadacd50079a64dbf626e3594957513aabbecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/5cdc7e3a50581b0ec50213b10d294eff400d72e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/c20df0c559b68437f6b6c5c9fb8687461c5d8717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593010.88/warc/d6e3caeae36ae0ecf67a4e8421e023b1e318e41a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593051.79/warc/31a457f9744d63cf996911453ba8e334864f5067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593051.79/warc/4f56ed1c66c2f814571c9ec469f9932d8759201f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593051.79/warc/623b4c35157bbea59e9dbae9bb6ee780adef0eb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593051.79/warc/793a1d6cd094ea34cf9312386632b3d03885ac64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593051.79/warc/affc44b25611961f991ad52362e870544a070ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/055ed2cbee36be49e56ba1b5501eef7335ebb563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/33b428fb6c08e80014cdf9343a65e3fd92f5ad3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/68618e4e3db7734c0e6b1e4b91feca7e1c33b7e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/ad35002184b731483db8c83e60735eedc2817cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/e18b7f7646a8d481a250b5834774856cc3339972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593142.83/warc/e6ec61e7daa66be03cf29c059a34b231e1a83699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593208.44/warc/048b0745a8e90af110c4748751bee82067530614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593208.44/warc/21b88d1a648bd70d4f4750b9ccf079c2eb2d9cd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593208.44/warc/39210569cc6d80e29d0832588e272a400566ef27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593208.44/warc/6f20eef0ea057fc10a52adb08c8bc5289f33d1ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593208.44/warc/eb3aa36b77d5ab32960932942bc599b620284108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/2c9328b8367a371535fe618a08e0b483207a5609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/406b95a150ed19f59e1de844758c6789b58c87c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/98eae94040fdd631dd73152b67ccfb0b9168538d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/ba37e7c863c7cc36e58404f116060a76de8beeb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/bd57aca26e3c1790e56644fc7beafa4f10604fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593223.90/warc/eabc95bfbe7e6d2dee8486498a60665c65e0f192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/3e39f03d7626463f20404a6e81e2cfb6cef869c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/73bc1e32282f7ee2d31f2f8c37aa4a1882d2d163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/9e35ee71004d83cf4dffa0f955cd40c4a041e18c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/b767f4e2413670e18644c6fbae6fc58326aa4a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/bfdd97be56593a7fd59331d4db62956dd91f6525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593302.74/warc/db069fc3075862afebcc4a6f09aa802e6effdf05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593378.85/warc/17753f6a45c25c21bbdce7e0e9d381689da52f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593378.85/warc/7d6ca4da18dc06df26d71538845acfd644b534bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593378.85/warc/87881dca97b15c7958e17aca8ac199902394aaf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593378.85/warc/ad692b0950648cab67cc521e253855a3e21fe688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593378.85/warc/d8b65b6a5100624b2ef9c4aa965a0ff108af5d32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/23f97d624cfd3c1d14563337f92e28fcf414fffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/50c9fee03f91e1cb013806c7f5f6487383cdd2b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/82347fec7c175d0c8199af89758c0ddb039e06ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/b6862177c10972364a558e6046a35c9fea2077eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/bc719752834586e59b88a08698962acc8cd091f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593438.33/warc/c3e12751a91b62c3bffb3a73560c41d1ee0d7335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593586.54/warc/21f78cc5a6603c74cb7bf59efbe462d68425c9ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593586.54/warc/3abac21edaec8c3bd5facf5c3b15e60a6479a98b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593586.54/warc/3dd4a15ea170b00567455f056c2243ccc21e5ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593586.54/warc/4c523cd9add40d40d7f6a3172121854e10c9b586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676593586.54/warc/8176cd840a2b014c23be443afaf1e59479f724e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/58fc2eae38391221e95061f4b0ac34ed83355932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/59b26ee69bb4b1cfd511030104c126b2cb12313b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/a0f7a0a272f24d07cb6cef9a9969458fd56c145b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/a7c82a5df3d177880d7beac14a554826266e3ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/aad4ce3127b857aa581c803df6650506151da681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594018.55/warc/ac95688bc0f40186ccd1dac20e93762eaac95f70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/1c98a83bbf8058f0afd0fb5ae3db9b9263864327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/52ae1d767115fca19c05e19b3336e61a4d6f5b63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/6a961a8199ea9bda29fa1ee7fd0d24abbe9420ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/937870d2e8802b15db8e5b681faa0af2d994be28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/974be90c2483dbf23ad67553d55ef87481f22be7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594675.66/warc/ef4d7a7bb5e1f2c50c68a8d91d16126ccee01b59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594790.48/warc/0cefe124c68540c8aa971484a1f45e2d5be2a4b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594790.48/warc/422e9147a7c23826811fb5538793e2f765371b4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594790.48/warc/542aec46c9b7900f50792946347b37320aa6625e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594790.48/warc/9638103ca9505aa568cf8ee23024c767f66d8a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594790.48/warc/d1257d29fc7e33a06d0862d7264b74dd85b9c3af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/1fcb066881d08062cb59d79c6e344d6d66cacfb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/852bdf66eec4a40a584239e674a95b4082841430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/ce039a063e788d371e7c94b91849141836245d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/de3d3d4dcb1bcfa1e080fdd148fe207cb3bc7ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/eb106f60dada954773d62c000bb59db94cd0a6bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594886.67/warc/fa067813c6fd142bee1f69247426e02dbdba6b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594954.59/warc/0a99da1d14c7c78b2083fe7b769488dbdea460d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594954.59/warc/0d19a2f9115711c00a212b501ad8ddf2b32aede0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594954.59/warc/10abd4dea9712fa41bda0b6193bcc58610113f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594954.59/warc/7c078e52ab60a299bfce6a4cfd59dcf2e4bbd9f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676594954.59/warc/9ca2e7288e33bacba8cbb905409027483d76af00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/36e4e920b8c6069473c29e55cde9d38470b97239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/3844b69cf9074322c2412cbaada5e31386322226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/45032324afe12184d38fffca537d5e9c12286e98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/884b2543dc7c238d1ecb5c795aa64e6ed820b453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/8b275a2f0cb83db4e3ad4186c315a41fd5f810ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676595531.70/warc/8d22fa3baee667ff19ab491f68d4b018f8af8353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/1aee6ed3bfebd25b1900d7c0a9008dc14e9610de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/35062030394041cd863e39067fbae8856f5b883d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/819260a19c1ba8e08fbdb511a0d65c0019e21307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/aa415c1c669c88176d550bb1cc04d9d590105bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/b56894917d9f77c59338d429d1d2122c03e4a45c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596204.93/warc/fb01036115128073a2b9dceda4a07f768c9e3874.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596336.96/warc/142149990d97647ad3b4fedc106dce624e15a1f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596336.96/warc/3a8caec71e7db800acce07d346056597b8e22e21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596336.96/warc/579e49e2a91d163094da357c109e20be88e4a634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596336.96/warc/703afaf7891552ac541101a0412a77e65d231858.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596336.96/warc/dc281a56c5e0851c24a4aee56028356e2c87ff93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/577e7fba098ef0fb4f1da8ee9c7fe8dd65f3b26b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/76ac5b83969c366b5c05a67de5732fe739cc9a22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/b370145958f5cd6ef2d6eef66700c75574ca164e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/cf524b9220ff1ac482b8f01e41d1200f6c163a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/dd7fecde0a3796ca1e7fbb2f33aa520c02c2b73c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596463.91/warc/e5b7e735fbd48d9d9d4fe02dda8a0f0f46dc8b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596542.97/warc/1871629219d5a49a708a67cbe7c7749768499c31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596542.97/warc/461358f6ffc0846c4892affff517b5afdf2681a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596542.97/warc/5b72816a409fc2980757ca0a4f40b90630b104a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596542.97/warc/67c80a70fe618fb42a6a619a7c49d3a5cf978814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676596542.97/warc/a5e1b0d72004387582248b6e392f8107bc23c20c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/1ffc93d2e4a9ace7584e526ad8c1c8a98ffd23fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/56dd90e18d716e42facc463bcca25627fc98b3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/a55e12c53afdc0b064b2b514a27fa7f6b349358b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/c5903c07278ef33a7334bea2f3d625227d88c536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/cbe24ed00c7950b146c6b1e74f010c3e4ba9746d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1531676599291.24/warc/e1893ea23b1ed57f63a0719b08ea72b97bed6f6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/19cb14d61b5acf4cf334690353e80bb72b32e5d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1d050159a9f4c879c124961d7d55cdd0c9edf515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/1deda4bbf2655eff8ebaa0a87f9456b565baaa3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/254ebed008e4d7c80c5932b1233dec7424d2a835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/268d904f9f5fb060deb8194b4fb3d50128b8a1c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/2c61588dfaf8fd0493436633709044e7c85ac91c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/2f1d9cc8a92f12fb4e1220cbcfdbb752add6accc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/2ff9d3b7019147fbb3b6a7df3f70206474816340.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/308ee4b9c39c9ad11e81de3fd4c407c82642f619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/30fd3007381643959124f078092feb386189b38c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/314d31251627a488520e19b345a9fe1ed1f36916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/3275629ad2551700147a64906fca34b4d64abba3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/38252e2f97610fae4c450b39dfc80a73ff02c050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/39a9de1bb05b3f604f411f1c8b6145e21cd1a574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/3d903b5dfa3e057b59e80f2b9cc29330cef575f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/3f2424a84d6676593fcab718ed530a7cf9e0b799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/43884a0f45b23aecae8f8df4d3dfeba943ebb329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/44246741c5cef6013e1e893236020a7e2f921514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/48d79266d1115505d032544834f7a337caa937c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/4a069f6380bc4100842e0f6a15d1c1ffd84495e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/4fddfc3b27d9a0a8a12efabd7b0cb93c03921834.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/521b05e692db4384e847d1b574ee84b52604832f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/523a39a7742d68aa44bd647ea36e3c986e249c11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/57e4cc170f2fec0913bb3d6fc7bbee61897ec0c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/59536ba150930604e26fc3f9b91f5743fe527dab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/62f79ca3cf687efa7668df0ecb2a2c1624f45291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/66ee9248c9e74358ce88ec82cabd2f67db84d61f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/69e77226b10d22f75531d2961635764c76861c71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/7ae3f43f78186f82424e1edd76fd315048d5f148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/7d1f96e6d26b9e8915362c285a075c29cfcd2397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/82ae31e5e1e6ad8d47ca95bfdd8624f99badf4c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/83095fb87d11c1f5b60cff175c3d6c1eb1f7f00f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/84dee9e5e0d6fea5a141ae02304430f506582555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/8548e14fbce1390ced4816b5dbd83612121a2b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/881f4618435e672b01dbb107404fa97a86855fa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/8906e3fb0d41f4ec26f32b5fa4bb0ceb86acf169.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/9110df6ac8b0c4ee16889efebfe674d0025ad978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/92073b7bc881ce14709f794e4d1f5a0c7e74e81b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/935172dd15aac4dfa8e996311db798d8affda9fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/9384e18f502ceb8964ed795693fd81b304c83a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/94652aba6bde1beb394e2bf75e3154a3dcbb99a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/98b4edbe8e63d5f7ad926f1f86564e3dba060ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/9efbbcc8f62f0924b8694481cec668f178f503a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/a0aed8ae9f57c09e6c23fb29ba74a15337e8406d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/a16361d9ba637155256a9450cd570095628e3451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/a88c3928e637d59e6917e072f4361759855fd355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/a928a7152cdef90cc62fd230c119f173ff1cb3fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/adea7c66dea4c1c1b79b45d69a9970abbac02b41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/af0314ce1ba85df0af1e8070e69da005824ff0f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/b151d1f4179abfc7de1875dce6ed3d1bbde21c4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/b386e70a1d29f9414a375532c29c3a79f6feda04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/b52a09fdb8b624b638a6a8d7ae54caf674770ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/b95b04366bc7c33705bbc9df6bf5bd783360e8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/bb748edea176784501fc76dd05b5e32cda147b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/c0242f7c779ce55c09c3dbacfc37d074d1f78ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/c5f46f913dc82c0f8252d08bc88ab6e4fbac11cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/c6fb4f15156860af22ea8b9c245bdabc599f12e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/c931ca54a6a369c0723e93fb8267e9fae9302ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/cd35a15d8a8779e13741fa6b90b341b7624332ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/cda392efeda4feaa6929b0c0300ae4244c5fb66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/ce51fd8bd6bbd2c870b2c09f8b9c429d888d7944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/cf5a9aaa473a46364c667fef4ace514e63765052.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/d05f0b5a435b856140b78a45dcf4ae45965c46d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/d2e92bf4fa732721b94699ddb945360f6ee2900f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/d8faa1c00778061f0b6a8b5420b46321c577f8ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/d95c47cfee49074e1d147c8de1a72dea13e0e4da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/df586fb5cc25f4115b7731ad4425c82c1d3d3639.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/e08d6da169af8b488359a32c2606fe6e94136c68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/e503f4a54f9d9dc8b1acccd5308c62168bcb7fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/e5cc949bf0d860d671d20866db8792d749dee80c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/f13442f8e396df7018aa2e032fdad8d6500c28d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/f80da5b9a5fcf4f72491a7a87c7e478556588a25.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-30 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2018-30/mix/input" + output: "/tmp/cccc-CC-MAIN-2018-30/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2018-34.yaml b/configs/cccc/mixer/CC-MAIN-2018-34.yaml new file mode 100644 index 00000000..408c0630 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2018-34.yaml @@ -0,0 +1,825 @@ + +streams: + - name: cccc-CC-MAIN-2018-34 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/014603c33d36868b99622c81c2f1c8d1c37df884.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/0af6972528fd846ce0286c40dccc22feb0a5ece6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/0d2889ecaf2a182a8efe6aed36f1e04767f588fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/0e5739d4526f9fa8ca0cd0687af731fa2489ef18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/0f73981191c29a4ceaf1faf4d42fff23f4f3ff96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/13285b3678f982f66edc6e0282f4a0d47608d0c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208676.20/warc/0bfd4f8b296ab815185730c0740b389045243c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208676.20/warc/5a22d91460f95f6aa69812a3ca6b3d71c369c4a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208676.20/warc/615418dbec465140339124423c7454f6a842cc5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208676.20/warc/7da5c304e2ce17ebe40836a10272927ac3f204ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208676.20/warc/d1f6d64248a2ef744920413cf80641a4a0fc1ee6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208750.9/warc/0e39c90a4f7e33b4596fe5f8285c50b52b3a6ee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208750.9/warc/4fc1bec5dab088236d43678822a54a57a13c282a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208750.9/warc/885e043c1d1483bb156af95ffe1585282928c7e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208750.9/warc/ba4e7cd545aafff0a60db43c43363b3bd22185ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221208750.9/warc/ee62addabe6e51d71778ebb3e84d66d602af25a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209021.21/warc/294c74c90eaef71cb6b862fce4e611b33135891f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209021.21/warc/499c0199340a085162c68491f958194cd22e0778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209021.21/warc/ba3df43f631d89b6f8dc9eed44f4c0537576f184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209021.21/warc/d8e68e2fb17ecf147d12477e8e3d5580c3f84745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/0800ef4e5995a582b717715d1c6f5e5907bd90d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/40c20a44e8bdac371ccc1996bd8196f72574f4ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/8485faffbd70d1dd1f6eb207d5c79296c572b4ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/93bfcc86993e2764527ffb3235d46fdfa65a404e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/eea5c95a97d9df16496d25ee51ea6952e9a9822f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/fa6b087d08691db0586015a00d097a4682425208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209040.29/warc/fb8b0ba85be75ee3290b9a147b2bcec73dadd702.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/14f8e8dc88cb43cf11dc3687502327719dae36e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/421f1ae19a57942e7b57cbc3a213714618861bcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/5a29797ea8a328141b7f916fbb378ce8dfab7270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/6732778ae551c92ac1b281bda9aaa893c417756b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/7e474f0128390b5d9a64ea00fea32f20601e20c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/aacebefb508b31901b721a9adaaa654ffcf27c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209165.16/warc/e8d05898c6828c8cdcf7aaa9ebfd6cae7b0285d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/15b2fc037a3a7cc207c8f4dd5b23dbf4fee83b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/58ef5658fc2ea5b4951972cc4e56f396225f790e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/73582732b56ccef775181ae6401e0a67f2d24e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/8e1dbce81c6e0fad03b45cd7e1e9f29108cc0ab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/fc208c6b49ed3286fcb1522e843106f3b68a7e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209216.31/warc/ff138e6bf7edce3b020b79324585c3065d3da9f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/5ca5a0b40bdd9083e009a2c1e09094aa2436a703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/64996c0a9320649684ff9f72ce424d24091e6473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/70720b64f097b0c7be86b48b3c7f2690ae97ea3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/794608eca3ebc58b6a1e24fc6fe2ddaf63a8d2b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/9c09eec19992ce1d7c979af61fb4c0478c6e7f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209562.5/warc/f1bb2516e1ab4365e6ffacbfd2d2a5ca0f533ef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/197efe8b47ee2f369e40653b36c34b50bf505c60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/2238919622f20d741b422528d1739c6bba1a7004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/3ed9cf34612731b5c0a493d1922362dd00b53b98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/422681deb0eb0cd0293e86e8c36e8677b167420c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/7c666f9b770e15c19f3c48bc7034aaf047567c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209585.18/warc/8d0be53aabcea6895c9972ab84752fe8737c8704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/3dbb8d65885872d5645917ac7cffad63fd01958d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/3e0964a490c5d6298c4c9afc798479f73506bdd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/890ac95585474ba2ad79b3ebe9e7536f12c75019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/a63c9d176cfedf2b950e5a92f8332993e259dbef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/aa66a4733a491d83d485d2c5d1f4c175ff2857f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/d3edde9b715ebbdcb01e2d6235bdfbba3e7a5cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209650.4/warc/d68afc27acd6f9cd278e0b5462ce415a41d6d635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/330c3bfee36949d595876f2ccc73a3e2dd6e35d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/7c5d288cecfc8019e65b33554ef635fc2072b722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/810e46eb1f0c296b5f3a45490c8f0c10dfa05a89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/81466973ee1d7ef8ec4d3e05b85374e1dbbc764c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/b1bc1a730bf84057fb177d79faa290ef2ab4a9f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/bb68a66347ebccc444cafd8fdc6af33394ecba9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209755.32/warc/d62149f43664b5641038f14cfa1305814529e344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/39466844513e1e379954648623763ac799a17188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/51f7f530b10373dee64916df09495c9a4129e298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/5aabaddab528b5ae085e15a214ef13b666af9245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/91e850bbaef153b16c24a1e491f37bb75c03de2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/a3e75950a56da8ecabdb0ddef6d0451f349eb348.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209856.3/warc/a89aa833724e596ca9be8a4c07a0d9b2906c41cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/1407183c4e46da1333237a45792f22a111963f29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/1cab2c18b3099e744ccaff20b4679173f983c56f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/2a1d1f4e5c0af92008e262bb7a18b074d467a39a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/994c2e2d234994f362229bbf23c1d78f451a0d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/b0cfb48e74d6ae2c17015859d0221505a250758a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209884.38/warc/fe43d766f55932b9c3eae30ed9d4c7b3ab22dd9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/2d41c7e36126877dedda6a184d82281d3bf7c40c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/5511c5b78cba0680b1543f2b1375f439dbffcc8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/6cd9b564357c6bfa47da5e97d625078fa409ec8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/b111c7c5dc2e1f872f1306c242e475e1cd75e30a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/d5ce460161dd9ceab315cabd244cf1aaa8f25e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221209980.13/warc/faa3992272771b6994132e668594164985c6943a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/264fcde4e6a1806b2c61729a9b7570fca3fba959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/3587841353897250a97e4f595892faa40b3d2fe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/574aa69b1f31740da4af4fe7b20b7d4890805c4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/90899faad0e7a424558e5734177ee8357f06c9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/b1ff9916d73b63d85aef1e5adc9c53dbfc1955f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/bfb8ea5326de72896871b23fd1aedde1257d1ea1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210040.24/warc/f1a9e63768d9aabb2b9645636e768d1ae32c56bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/100a85f67853d243243317026d28087e8bdd552d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/1761ab44ec5ce3bcbf13a0fdda28f3f011e93c62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/528863784058dd2b36a8da20de86da84971d66ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/7402111154b2fb966ae8966de18564b1e63b55c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/92a79861e4b1f6351496329e03df79838655e261.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/a0bb2674a41cbdd08fcabeac5666c3e0d5a7836f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210058.26/warc/c7d4afe7c1a56d0dac83b97c4883fb33f2c9ea14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/777ea6276e0415d9aefa09cd04ae13fd79c229e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/acf5ceba419799e3d72487fd908a053a24ac7ea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/b7bfd49422f5161192230904761e9d8877d2a554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/c8d22b656f4b7941233d1fd89c141385e9751219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/d139ea64ca8cd0488ba519282be4f4d51edee181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210105.8/warc/e8c36a336f249af51ad0b6428d499c18e66a45a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/0c37607ced959344743c178a931c52e04692c5e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/19cdd052a75a9e18cae9f021091f51f66a8d1e96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/2f9b113606772e1ade61925e811427e175846af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/736ae3a50a454d24f223dacdfc129665875f03b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/beb133f4d1a44657db3996ca26026e9f0ca2f7c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210133.37/warc/eb005872278e50f4075d778ba3f5a2c7a3e4d8ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/16e527342f08352caea815713d3b6fcf40053c24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/4350671d9bfd58bf4b068de37eaaa852ef320bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/7c3f9cbccf371fb81b2611046065ee0085b0c14d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/ca0db36f610026d650b400b182430133d8286caa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/ea8dec53fc6d0408676c1b634493ba60532ec613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210243.28/warc/fd4430c592f8aadbb0accda15cc3a105391882c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/218c5b20b6de8e759502979734d4fbb72e1b3db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/31ca793e1261a12571715f2d483583827634f763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/59b201adf2bc7eb3e7a3c448412d3e17a9113685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/d52008e6b17f92f2b4fc3a93b53bc3a3e618ad2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/dda9c5d424fd341bc599b5e7c6f27b0f3b06a888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/ecd67c93260f2d3c6ee38f4eaef3736e8294ceb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210249.41/warc/ffc5e20b1aaf45af9e0784bf20d7423b765c69f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/27ca1788f50e1d370da31e4cb83baba2a82f0cbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/2ce2596d93469e3199a33d1bdca6ab687a71ace0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/4a344cef5dd74efad086a530f3d76977786caf37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/6c4ef12a8b820451dbb371c81dda121a35faa1ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/8ee96874febc3a579890f19a32b390d9b50a9d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/91c4c7c0d05cdbbd9bc88451406994c00ea5d7e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210304.2/warc/c516a57fb06b698ace42cbddc905470844497f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/03b7391a04a2b60dc6ffd97a7d74ab2b8a47129b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/078e9e76f5609e0462a44e100bcf48213836db56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/75d69e3c55500934516243893ec0f31479d12df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/b4381c59d11b4347ede0486ab57ceb64731f2d30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/dc510a359d87bb17c67b7b981106fb9e5e724826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210362.19/warc/f57e6a5b2fc4b087bbdf54f99a6120af86cd95d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/0019407330731ef16afb79fe7c8c38f76dd11faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/1c1ddeb062d036be7ff3bdd29b111893d21b4acc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/9358d8c40a8aa393aadff30b37f42cddf09dcce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/970874162c50251626ea78304902117bceeb2b75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/ea3056bad7cfb843335d8aa021efdc2ced30729a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210387.7/warc/f2633b38ab1cd4be121d2bf7c66d32ddf0f54385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/02fdf71a3ea00d34b13017f61907bcd7c82aa919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/16cc22824c5df9e969e469133b855b2130ac1cd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/2bdff857cc6c75e2c108fec77af40901faa06aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/2bffb246618759663688b16c25df7946cc5ec83f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/9be84e79892f1e3441bb2c55189e37095b156057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210408.15/warc/b6d8a948e1f5f7506670ca2c5cd43b3ac39b2729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/1b04008f83f1269cd949014716b8c99ce7546667.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/42979bd759b4b3ee0c22eb241e0cc7a18bf6e196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/67260385d2214d1dd5fbd083c75e878a3c4dcc75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/bfef6c2f9ab0ce6743bb216623aebe05aa162e34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/d282fa8977c81de9820b8da466e74dcaf8a455e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/dd3e6149632694482cab24dcdd4f6f16fcc779c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210413.14/warc/eed4d11ee34f7e599efc072930e66fdae1b6f645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/125676e5a5f33c1a8588f147baf13a9521235f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/2409a22afd266f4a75d0ea89ef7cc2f6c3300126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/3a41ff0f787db92e152e2ca2bf5fc9a72ad33460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/c651c9e4942b0ef5862b9a4c169545ae6399d642.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/e82d4e9de88e3a6ce49aa4cf8e8f39166cabfebb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/f3cb166475e7f831bcc6b5c3357f390aa1851276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210463.30/warc/fbc94bcf6013404857b954e2e9cd1821872be343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/3ee45cfc333730c0bc7bbba29a82a2b789c23553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/49c36c486401c914c243a20ff7d8e10c69a7dfaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/b1518d6d609927003c7cfc00037304c45d7e2ec5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/b188d0484f2ae3f5d44f99237aca92adb67fd782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/c16bd1caa5a170b64e460ceea2d61f2355fb968b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210559.6/warc/fccc632c7053a748745bd2006dfb1df014c2d116.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/2729e72420581750cdfbe38b5b5d518c40454520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/2de6b6db2b33199290105944f148bfb40f40ce5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/3f8b39509f41ccab377258ce46107f8b7bc4685a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/61328b9c9d765a10aa45c463ee6c687c5b53daea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/6a79b3612a5e214f83b5c69eed636a1c5e9a2d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210615.10/warc/f9c34a661fda7b404e6dba006f05d3f52223b01f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/2a359b56bafccc565c5bd4ae898cb1ba0daafade.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/4ca5540114e11ded8f2c6061eee52c756d5e5a9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/56918fe084f25c64312d3f90a7d6f7227862caaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/650ac4f0d843975eda88bd2a704783ac2bab8bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/9e34bc0d05e8f4070f4704adf83a01740a4df705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221210735.11/warc/b0bd3b58a1c5c39a6c7f343187532a0ec5bad1f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/03adb75101f4b345aa9966e61feda824b2182006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/0a1abaeb63fee5c5092cc6b52ca9404996f5594c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/1072ab4889ffa10cdba5bc201f60cbf2d5705f8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/31b4e98bb4a7b2ea1e1d9f5cdeea845367e30528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/9f0d18cb857dc61d20014011abad08c6d10381f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/9f9297c03d81f00be46cbe120c0bedca16b25f6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211000.35/warc/e3a3d414d9ac3d2be884afdae27a1183129fb973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/0d7b64cb5ddf22efea27211eb520c216afcd7b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/16b995ac56b24b4b1e5dcb9a7e4025657277f8bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/1d0cea56908f147c645ea2e980eb37e0924a6904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/a165590c1cc89e42998b8bcfaebcd8aab38ce090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/a8548ab26afbd68c8c576e7d5ab50947a37e4093.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/b97f15d2b898026d61ca54ef6e44c09b8a99ba09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211126.22/warc/d0863851993760c5eb43f181f8ffa0f95efc16b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/08ccd02067785f7f86af97e465bf0d94b1d22709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/0d8887175cbd03147e777dae5a84886b65103291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/4de61500c630157422ef415eed012d85e2306dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/70fe0a003cbba07f66d6ea49db6210ce39add4e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/c2f6ea12044982aac03c1724fc2010ea260c0fb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211146.17/warc/cb30cc0f00771a529b56dc8a57fbd8dc2ac98550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/35a84ec6659dfbcd5b388dd61420953f02cb0c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/7baa8b8450d95c487c3684fa6fd6d333d6d93bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/8b3af4fa76b70e61b02ad5e1b39360944b119afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/b21a6a11c6d40c8f948fe87b48a09dc157327a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/bfb40f453576caa17010eb9a040a89b5313f1726.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211167.1/warc/e646894c430fe0db3f870f1311bfd1a4c0296f6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/3f3faf82fa9052d2f6c022f7f4e97ed20a49beea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/445614c22d43c10723ab7a1444074e52fa98d16e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/5ed3d3234fd38ae97b3eb52e196e7374fe0a1a7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/64a6d199a4a1fd20d6acaa90c58a7b97f8a7c2e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/85d209708b175fb3fd47ecfc12475ca46dd24e9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211185.57/warc/f23c27bf1a1c01c3d59471ebbe9fc3f90c385a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/246bd020b39ac5fd72819fd6c7eef90e5896b24b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/53c4a400a601471efc70b1a86b29e4f21c9307d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/8668aadd30500d2cbf34cc60d7985236c3a1452a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/9aa7eb228df3cf3fb7df8de4e4eac314306389f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/a6ac6238d5cd794acf4fd27959bd77d088803f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/bf2ad689187daa2c68c08dfdff6752ab7ab5bfd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211316.39/warc/c81141d38e8e8d35fc25909b8de21f054d8fc195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/15143916856a34c19a1df96b402dc4375faf40c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/1cd29d271083723d2ec8f2e5c572db2a8890bbc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/329e5a6c37ccbcf51a1ffe80891d6fb10d758f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/4b6b49104a0814406b2c763950f109030db8cc89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/891fc99a1e91d00308624f139a3e5d767a8b4fb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/eff7c73a99b7b138497145685b5ee70be2251f0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211403.34/warc/fc85afbd8645090738f42ae05ad3b4f5941712a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/1c63f12c6ab737f1a9bb5a33b4f2d5433505d212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/1f642b20721a4f44d6b2488b900c669a504f32f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/5e7c81323fa2d811f2018e7b0946b3bb2808eea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/92c68295ab846351256ed4f9fd1f8d741df5dd00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/d35d378c2d609762f495016010c8a4bcd03dcfa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211664.49/warc/ea70d5560dfdc2a927e90f0ed90e120b8e18910a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/164e71579e8c9ee21829c6d88c58a4a98408546f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/333d11b928e173c4776209ee8f65b086ef476e61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/406a293fca63eb898e6a62f4273ccfd7e81a56ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/bb27068fbaf8e9c6069f36706304531f9f7697aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/e87c59363f6a192b4bcf39e0b3d8d2768554c225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211719.12/warc/fd90e14f8b865985cd51894a75b4b2eb805e1dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/535b9c5f2483279ae7cffdaf1fe9e5f4bfa218cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/5994dcb64e9236825612bf70db2faf751ecfcc72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/870baa042bf96358754d3546e974f1f55b68348d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/89afa8f35c1c99c60d8dc5249e5b7716714539bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/9999eb29d1f55567ca9a61e4d7e6654b0e4efebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211933.43/warc/f3ee72cdc6769210e881aa002b13537d81c44ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/1cc77bbab9b537ee7afb3f0f8040215a70fb4b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/3315d853e0cdd326c70e574cc280b591d7c7008d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/9c124d440550b64743587ceef57fd26902e1d14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/ac418914d9166e7dc4e3ed73a35b9a79ad011a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/c3ff26cfe6c9d7bb8fbdcd0358a64b74ec6aa32e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/d41340c2536a77ada94ca534911729fd70086a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221211935.42/warc/d938a74c4e014bbcebf99c87345403ea6983fec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/454ae4dfe39ee5dd1396711deed2c4a08b5bf1f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/516d4efa40a79a235631558d6c60963cf9b81dc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/6817f88d49c33e7c1186738de9f41cfef43a94d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/762704b55ba295d12e1d64b8d97909a495bb45d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/7bc01c10cbdfc620ed38dbd3f948b8c45b08232c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/b44f624593ce4d39d032947c2260586f5c494512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212040.27/warc/baf3cd5c622710f34417f2fc72269272c134342b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/0ed4bc099971c1885b6f77c537f702e5314d0e42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/33f62a2bc115aa7e51cf929731b82fd13319dda7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/445e4272c37d22018842e1321138ffff19db257c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/b2291ddb060beb380fb869fba8d0ef7d0da23673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/e54247105faf8720ddbfe324b0c0f5bead8bb6cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212323.62/warc/f811214b9056674a97ede85433e2fd597ea96d48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/04cb96649b6e45a34520839fde3ceca8aa2d4480.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/3dc6e902929c8459cc2ade1346c6711cba2b936d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/42becd5805a112ef3b6b787be354b604b8dbb7f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/5c9503ae9567a71b2d8f06802e89ae9494114b43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/69604da3283cdf5943422a6280290e385fe90dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212598.67/warc/6c6b0c1160eb108fab219b3429927caf036e35ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/101b30a14becfacf09ae23fa3eae96af87491502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/209ce7e80c9b455c61d027007d14873c9774d4d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/9921fb36149496c48668d9a4a436bc868869af0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/ae404a301f7348b672e3e9a003af5ba9ef647296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/b86815f77c3f567a2f0284c0c4a75fe4a1a3ae19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212639.36/warc/e2f37e16ad41b53f52ea9e00fa94aff318756d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/27070cf6ca3e8d1f03e540779170e5dca82416ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/3698d498eabd69bbf963fc80e0d0c23ae4185bc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/6a8f76f8bcbd0cfdb9acbe5ee01eb9d6ae2e6cf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/77e791608517030009c60bea57c7481391660a69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/880117900115b67c58e6559bef93e126c124c0d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/8be1cdc36ebdf9391909e44ab1a9219fa2591d45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212768.50/warc/b60062b7ab6b45190f85eb2794f43fb3e95ad720.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/0579bcdfb1202418ea752ab0ca14fc99599ff163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/1d7fcd12109a430d9ea18679081deeac21a9e561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/3fbcec32d1df8da0eaeefabaf4ff1304d72e66e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/4513ba6ba6aabe01e75c510251c7fa8c076fa4d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/85c73d9a7292fe6dfc6b0a78b6d3c2ea1d3a4504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/dd861077ea8febfa1e3acf27cc55d5938d96dc6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221212910.25/warc/dfde475d70c370f1b9e6b4a3b1c9232a7c840669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/3168fe192bb7dd1d175a969b1529f516ae830063.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/3f5ba844dfa51bda987122061d4231a271edcc0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/3f970df363b2d3a5cb5d65520e67198d3fcf16df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/63a4eda6be4007413b1f78deffaaf8dc0600ca96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/da916d811a05034f5e1995c2deed5d448860fc81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213158.51/warc/f38be1719f5e79a1a4ad3e553a741f64aa33efa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/11958a9474e5be76099ac4bd7484855dadf63c40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/30e43c2d72c4d5b8d715f59e71d95b1beb304b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/435f875862b13823ee145842a36e3475fc8f72dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/6467dccac6c2b5155685239bc172e2b3df231017.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/c8eac290ae00b980b7355ca4f4e082ec3c5d5d48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213247.0/warc/ce57e01889b96338937ede10db1e89d56c15c3a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/3aee5c69019fb95febe3ddb65accc4d4272f2e77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/4d57e683fa970188e18a9eb96721424befd7672f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/b6c5efc0c46996ba0e432a5f34f0715c9194d105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/ce72778c166db568e22b216b5abe980fd8613d72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/ebca113b33e109747b16051b5727a18b15e30ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213264.47/warc/fc8dc60ca7295cc0a1dd8651d43055e33c5e94f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/3e0fcbf1a79080e25e83ff44c17c8322e08ffbaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/5d0f3b6cf1db6178082f353caf41a0d6d1a92d61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/76ce9315e1a52695cd3e117740bcf16c5f4a25f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/a308ac05336b37d33136dd2561015d81246047f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/cf07d8290436c961f3cd968e45eb03822dad6dba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/f52df3354490ff9dc471b68192af2b02ef2b8dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213286.48/warc/fa743878f4b86f878d93134251f6b3c46006d53f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/36485ed34818ce5e68c27e1a94a46e376e39ff51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/3fd0fe88dfe4d2f8d3b084567544337a1e998ddd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/533dc606a262095950221522099729fda98c492d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/7addd4dd4fe27e8bb45fe7e418a84239225f9f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/8d8a4aa0a3b8e9e5516dc4a3885571d5ef2f75f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/9c95394b794974177a611470d65a2437174bbd27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213405.46/warc/dbca028e56adcd256e49164a5b2c7fdcca4588d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/15f66b5f691adc6a8043e05a4d084b8a0d2fbfcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/45bfce31db77278a14c721aea2e6adb7fd83e340.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/77f2bf0831fb7ae1b6597f6f4ebcb15435b2dd02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/a499b4a36e3b56be3f95ea614ba9a60c1848553c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/de2075f81b9275c46fbb7f6168b492a95ba2616d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213508.60/warc/e26b4856e4c3779bab2c25fc06dc217a47719101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/037f4e81c6d53a3fbec732d00d6c9c0cdc34a43b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/15afe7c33eb0fb70c62dc99283fb21aa95a3ffd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/4a34894448cb5c2d5e93f5b7ad2349ccf874f4ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/5135e902a65e8d939cc388dc817466a362fb4749.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/bae20062bba6f2859900361f64ab598b849d9182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213540.33/warc/dfeb3028ba541cc7a8727ec67f3251121f11f809.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/3d553f9c018ade5d3c6022ece6252e7c70abcc68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/49b9d3ac0ce5c473d065706fe9771361882282ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/4a8c9b6d228a2a5bcb15f929697c6fff5a6d2d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/d30675987a659105adb6406eec3468a658af4f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/db3ffbd4da741cb7009bb3dc79b2039b50bd8526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213666.61/warc/ec88d6f15f06d6fede6c5276b6a472fe5c57c9dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/452d45f065259b312ec687ea379688bae1653a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/5be448ae7d5e6e4444eb62c918720187abd2a20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/6b60aad3e089b1c76f8b5c21a37d47ce8e361b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/87a87fb989dd711837af3618dc94da6afaee8a23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/9c1a1c82d852c3483f7b18b881057cfd74164852.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/d15f82002daff025353c6542ff81feddc92185c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213689.68/warc/feedba8f90ac15150fd94986810fb84413148e24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/213d6f3ab41927b1ec2a6d1186b698ae3faf824f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/2207e9383ab1c8649af6d44b0bd3dedcaf47c6fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/31006970bd7528235f30e3caa04b774413006228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/3b4625eb20b79af81fa887b934049224e4e7904a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/d45fe02dfc505c91cc2ac3ca3f49b30708b98865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/e25421aaf4f06220d4169f8b91a7577e9155e3f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213691.59/warc/f1aa79071fdc8a0528351548b4489c6c8b8de254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/0f7582300ca091293fb29073c1d79493306e0bd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/13fd5568a678f17912d975e52e378b5463d97d31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/1d487cc8265cbfb281b9ba8ef676d51d9a5d5335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/3770b7461b5d65688123be45a6e67df671912f7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/dbb5600be0020263e7d3f5fe944a326c75a208ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213693.23/warc/f2b2e6b7618db6b4db08c88e534e22efbf8c2177.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/29ec15f8e3854cbeb1d8c31d418284128211a8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/475bfd2ccdb402e9f1b80bc2851e9382964795f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/5622e6ab42cc72c6d05d60eaac4f293e6c5e81e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/609465ce4125357069adc1cc23189f99ee9dbdd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/93a3057d75927d200968d49587809e3b441aeeb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213737.64/warc/9f937be7477a777f9056a71a4a99b7b7df374d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/38b1e97326a389b79b1fcf58539cab885332f59a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/454400857553b8de31598e843f0762913fd8ae7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/49c59e1f67da143722f0d54ec0d75c625a51589a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/8e1de8b39b6da313f0f0de11ff5118cd3b6dc8ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/9f31007a8603dac0725c999cae45ec5387143684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213794.40/warc/d1d83405617b46eb7fdb966cac448f3c8bcdcdd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/08ceac0e3f83b6924a7d15213299231005931111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/2f95cde6dd42d4fcf25f0661a4a55ef701d2c486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/4d1c6dcc70b44d0574296960a1976425fa02bd17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/4ebadae52d4f780ec335c42fb1309813265fbc8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/6057f0aec2fe61d73f342b8855098a45b89a7ca2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/7aeef82a445101cc4a5ed0d1053802a7052c9fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221213903.82/warc/ac745054207fb061cf457fa5e1a3f261dab5230f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/085bbd247597da7c823d56bfb82d801adac55367.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/0d74b6b5638025accb7895b2ee7cb8856fbcc736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/77ed1b450895cde1c0a8a609dbe31e2a5d61b832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/7f03e16f838688c03e0ec6b3651feb8217a9e47e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/9e0de0c90750c64f7d51025849ef38e79daa4c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/cc97b6e5047c5be9d0246d3c273237502ab1efa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214538.44/warc/f2716f39ae7aa4f4d0563b916e2570764e44648e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/17c1d4993ca6cddf8d11601c38bd648309405437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/71b5f284c1dda6d35699b9b02153d75e72215952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/81964014e4571a5cbc0b41503242cc546e029d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/8a9ffe7a795b28e56e1224600c92a0834f115845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/954ceca5dc201df1916887fe3866725002eeda62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214691.99/warc/db97538f550057dc5b4432bc8a8b88236251ba36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/33a5ffd3a09d1d2159daef06f06c20f00503902e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/3b8a8dbaed805b4a350cd8fe0e97d0e093ee4325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/aa24f9dea54fe86d9e6adb7a45e77c43621cbc1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/b2319769241161b4c09d0bc3bf4f1f48596db1e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/b9689b86f8bc2b88b9105ab5c2150bbc265ea3fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214702.96/warc/f6e31621f58a0b3a5f7ba02eba062548c4d0d718.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/1c9b3c46b481be4d96e20af1b60238dde96a767b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/93388002a9bd84b69136c90ca730920c5745ee17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/ad86fcc1f1313b7f79c6bba36fb6f56b2839ce7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/b4bde06c036963869d84b1f7d5e548638bb227f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/d83203c585c916106efb4bf188a4169b916ad4d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221214713.45/warc/e5958af5abc8a740ac20549730bf42a67fb75693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/269c709082d106e68b6973883967139ac60eb3fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/2c251124f58b81d618a8d0ac61f35d27c1192673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/39d7dbd3cbb5d1a3f76c4c0055e9b74816c0cfca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/66005d672cd90fc4928cefd4ed60c68b9ac5a57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/708ef09c0bfa474d540a525fd97f12de709d1b1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/718ba14b2a5f40e0a9be40c58d31a066f59ebfa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215075.58/warc/7cf37e6a3c853ebecb99a8126f09708a48b1e92d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/073cb3c186706e953655eaaf9afa2e17b6141b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/191db966e25fc93bd16cbd354662bde0f6602573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/a521208887daa78bf381d23fbdf608fe73479f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/a52320680dc72d3a65b3861149e7c04756bd6410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/b16773325384c4892fe1a647f49ee137347d6143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/d1c5b26c77cc7d6fa337638ac62c4fab00af5aae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215077.71/warc/d2910c0877807ecd69977a127885c49b301e3287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/3404e940f0680a05ce1015134139cc6c6e851d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/4a9e1c387d0e60d558ee114ef6bce7529afccd5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/b57e91c4809446726566efbc446200cad099eb96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/df110dd0fe376340d977cfa4988d7073334d36d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/eed0ace4a5d4a3f87e7c4b30cd5fba7527c99f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215176.72/warc/f880994b823d0a323d0a50a20cb0b79abb5c4808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/12d0b640342354b3f9afd353fe8b1dbef5534e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/285428b0ad63700defc2da3b8e7b37004d9f453c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/4349537c354cb19635b4fcd7e04131b23cd578d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/a4f4d7323e25b691a3a9b1832f650c7b5b558bee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/a98198302f546cc5fe5829dc2904724e2bcd79cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215222.74/warc/c276802110c00a4084d791c4ca4ec5215c0ed873.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/07e1ebaf60540b5cacfecab42f35b6ef0d45b140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/54c2adcf7f969439d80967b3ba8209a50a466cb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/7dd4ac024fc82c4e58a3e7599ca0902135b718b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/7fbfc4b3f3f8a9b9f119036d2585c4b05ddadde1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/80ca28878c02f4393bc24c1547c3c3776c164086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215261.83/warc/f3f30155f331a5ef5f761e3b4e875d3aa98516e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/1633d04623b4ea7cc058d5b2fb7335a037280ed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/184fe8c5e11b52523e3ef6fe392f08b6066b9c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/7212d82114fac91a9cbbf4ce3005503e7bee7623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/7a819e1a2c9ae434f0b96e8b97a2411926805637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/a3456facaaeab57fa70da02a126962c2d8eed13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/e912b2a98e598c514a11b387cef3c23202e349f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215284.54/warc/f47f11bfea6ac9b892ed7318e7581bf86e35f406.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/4e47377686af0992904476a615a65764c98e68f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/7029f4f9bbc964b95a161a3449ec9f33b58f54fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/87e9f96eb700704c0eae89016244e6d0c0551c69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/9861977212ff9520b22ea46cb2d193d2ee38d542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/b9e0b588ced83c436e3b0ed43b77f8dee75b5e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/c1b7ab37558dafe82f769d9466a81709cbc4f6ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215393.63/warc/fe34e1e359ea44b16bcdfee3e03e9abc3a56c91e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/05d7d83c4f0528d93e087131535005399ed2c87e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/0e89780f7c872527ccd1f8ea5bf065c36851c2e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/1d8f2e731227ec15d5a9228f704f0bf764995e7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/93b3577b0c4cd41e56a55a01827736e2d8c85e2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/c7031de8c78cfbff0d0f5c19c31f93397c55058a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215404.70/warc/f0fed9a21bf9553e78b8e3f99f3269558140d99a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/08d313d59186a2e8d8340d4fb8355a467a52b273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/530cb0e34f7b906ae646512f0863f36026829007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/845dd1fedb8cc342dfbef6e82dc4fc65208292c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/c47275595071877ad693669845e5064c07703f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/c6aef87a0402b6b1da20ea7407ab9d34303f2ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215487.79/warc/ea5806ccba05a270d6ee181e03802f751a1ef7d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/150551c219cd6eb68091b773c761d6e4a61f9bd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/2a1ba4e4ae50f3c53e93b9053b30ecedd6fd214d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/35b6234ef171b17980bcf0dc7b4c5e4af106f9ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/60139d51ebb0109b5a75630c4c4dcb42096283a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/a33bcabff73b29dc036a625b82cfe9492d18ec2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215523.56/warc/da028d0fd3e92bf33263334c2c899fdc4b74d428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/0e48c6c1e0ef6dda211082f7f3398a7a845691c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/0ff3deee580feba2dccfe604ecf6f68beab30673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/3ecbdd30becf5af2f156fadb3ecea1c5eb39bd70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/422812be562857eb3c69cee2f25d908915f9d510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/a26d061cd0f3be13013cdb53dc8778489290a4f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/a4784e0b47d9abca69d1bc7ef4c8ef322f8d1588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215843.55/warc/d8dafa6e459f1a95acfedef50cb62896a1ebc022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/1a12258177541e114ec754da30ce4b757cbf85bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/29a5fc3a4ccd33a45da7b6dd583c9483580fad7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/40eb20757334190952e394ef07fe1409523c9ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/4a0f7566b93b76ea58839c6775b83699f1709f5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/8192bf3b132c68da3463e45823b5da34264470e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/d30aa88feede6a443e22b806f41f625d32625393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221215858.81/warc/ef94e575d01d58cebbdbd823233838daad429b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/2da106460d699e960d2bde0a5ae40070182390a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/4c402d7980458055ddbd57f9c20839c899b287b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/5eeb1ff5a09059996bc8cb2ef0c85ed1f9d69a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/7c55c48da845e72e138cde75f7b32be34a31450b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/9ee7f8b69977dc6c708a177e891dbd42577236c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216051.80/warc/a3131fbf3ff78854d871310ffe1eea6f0090bbc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/2e6862b7c8231650f158fa8a735852a2c65b2d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/4041333021784e459f3cee06a0cd4bfce6de8317.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/690cce97a580b67f0a5c7fa6c3f18581d599f945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/6c43c7a9a914912b9e92222717eeecf8803c03fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/a51294a2916e9e94d9e226b30bc963e36fee2827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216333.66/warc/b485f244a9c58a578ce7852c4c3c4f424ea14364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/2a825b5c58380392840679e1ab97d1adee472afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/5a7f9ab18b6b1e0048f73d00a9ae30b5059ac66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/c245d46541f55bc1ae17ad35b729cab4fc348dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/d2ee560650db3ffa813c25c44760334c58422fd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/d7f7e378bda090fc2e610fe8d3d4b66678e5f191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216453.52/warc/f8c84b05c1f85be9e7b098d072e2225d3ca606c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/23c4fe7bd0640d7ed83017ec4c4d971c53b682b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/47105bb9568c8f31f13af6913baacf436a66b94f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/6570fa068a6e6c3abbb4c9e69cd23d50233eaaea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/8f6a10c52af23afab0aad08441b23e5d3bee98e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/a01a17e4aad91a055e6756e1eada42b06a9aff73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/ded67b73c15c23d71ba301cf37c68d0f86922e8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216475.75/warc/eedf3fbab6a61c74ac7bc45b6f76e7ac6feb1535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/499e28731ec4e29b816e5fa004148d61d976437b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/6b10ad52347399325d3a8b957bb0d4dbc5aa2492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/7b61342f15887414fa2b816fa27663e876a20437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/96f47fccc308c8b8754486e2f1e928e2fd9bf864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/c272a50b3a2a570701d0a05dc6538e3fe157e549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/d3030773aad5be96b5d70da97b31fc886e286300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216718.53/warc/f197986ce87610726943045b00b392e6b33b58e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/038bd5e3535a568ec15900cf9ec51e20a4e5c1c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/324b9700b49dc1b9f9365cf17577756a83193bce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/48ecc0e74f3913e8ab30dc4281a62df9b7cda859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/c49152a249a033e687d49ffbd36d827790c60c78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/c96571b3d14c3521f69c0bba056b2a2e3b5b357d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221216724.69/warc/d8d3f69cbc1c078d1909f06b57a3c42fd8ef0adf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/0b1e10622f21a769ef1757d4497c11ff2c67963d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/6c6b7751ddd4481a0f490244af0adca7fed9416f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/a57c68ece9dca1cdfb90c3b5e73a00682904c2f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/e9fbd6771f6f137287ab6d7a786ba53712bab4f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/edf465d649229724f4597c39f4c2ece6cad6bdff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217006.78/warc/f5e0e5f7155683d3d5e4d3a7cfeccb87c82a8005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/03c8f57579ed676fea9ae176b915fb69e90479b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/37e30bd6fe20922e70baf40205ca6c8effc41d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/43c1eee8bb6cd7545047bf793dcc2498e9a56d66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/605d48f779bc00157a5110faf4e55759d9aefc49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/88f9128ce91600bfb7fef4669d2a8de941a4c11c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217354.65/warc/d46d1e9be69fc0268e0a97ed40613c0567760b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/1f8f4335c81412044d9f5c8b0fa2948f4d2fc3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/3add7c18cd82484be5c6f19392e4a042516a877f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/42e684359a0355619c7ea89ff07bc15705995f17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/4438775debc1c5772ea584cfc3a87dff9c8b30c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/6d100dd9037cb767cdeb03e2498948d1c29e72eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/7240a4959e81d25620e81e36968c294723e9a096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217901.91/warc/b7237391a08ace2dbb30b7212ffcea3ce9b06d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/07c1bbf5135ff81c7c60065b5d9904ee951c7f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/341e36ee0654b18aa6fd8e54ca006f6bb3e27172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/355a71748a5ada74cc30f8bccb56cb4058e54501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/4a94e28c57677ad8864c359b1d489027007aa7e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/6cc691cdfabcb31223ce850748d8dbd0060401a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/cb80a94140f2eab4d461ea35aff03505443ce598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217909.77/warc/f9f041ede0c5583e6436e45caebcbfa3ba07f2bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/2e00c4deead135a1105305cae7e820d297f2ce2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/38ef4b309945c339e33f1611879bd74ccc5ca63d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/477b81de9d66f92c42fea15f47f00d94999c8d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/4f08a1bf2e202c289ad93b00d52f3a821fb2ef90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/58d5555eb85e6c03f3f892b065aeefb333512c35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217951.76/warc/80c7dad860bfa037e98512f24f2572f68842a6fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/01e348e96dde7cf22429feca3f3a5246f55d0c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/2a1b956e89f41c7e04895652696038a21db8ae42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/878cd71720acbe6c7bf467fef8ef844cc456532c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/d7df291827289bd034712ee1b6d192780d551b21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/da9f0fbb3adcafacc53a45d3e9bbfc04ea604022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221217970.87/warc/e4e233b59773785d64f2048dcbf7f5f7876e4614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/0585bef2f62c9dd22410751d2a281805d1141c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/2cfa3988f8c451748fc6b59559312cc703f5be68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/425171808e53999c56911427f7188b1c22da7983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/4b6ffbb89e2c09a608cfcbd8b36afbef1fc9fc1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/52eb6e5a9ecdf04184829924dcaeb6adddbe2887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218070.73/warc/649bafd4f12022e0858850d5328f12fff399eef9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/0ad3aa0f39d45c6e08b036a507ecdf3d90f05ef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/0ff75cd33bb5daf3eb99465ecd72eb3fbadb53ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/78907e1d896132e5007a8b994a51f25e0cbf4f56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/b0511a387e4ba7ee1625a051295f72fe723aa910.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/bddcb27faa6d2a2d27c80bc4e907a152cadcaffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/cc80be7b2031a295ac7d2919459286f04a5807a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218101.95/warc/f4abf32103e98c5adf7a47b85070b7209403ffe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/01b6ef1d6c651fa551127873a223e3e3c70b7ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/1864e497742e778bd26072ae93fcfadeaac2fc25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/7c7a5f704021faa0fbe65699b5c83fc65607b4d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/a7eb29d1511d8ea6236b0200ba5915a767206a8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/eb22d5ed81e11557e778327de488a6ed8a70722e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/f61786bfb5709afc929b42d65dcf2737dd84768f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218122.85/warc/fd97907afea58374b382be101076c5cc08d143aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/53ae84ed52e09df40a40e844e19c8d5888198309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/6bb84070759edd19b45c3642c1dd89d74545fd35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/8cbc855d468b45d514369166efbf5171b346afd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/8f6ae004d81be04e3181734c3905916166a0e184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/a8ef4a621532760f4d27410eea935360ace53671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218189.86/warc/c6b222df2ed0a30c779c2559ef6b97b8362204a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/3e433bc3662e1b8c1a948f9d45bc8d61cd6b5538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/b59457945d2e331969452baf370ffd710f0d89c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/c18d685eaa7c0777c03112673ed8d5b3afdc01d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/d5eeba62431529bb01fc46aec985b78883910e8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/d87408c73c49feba9621cdd53ca788a5bf0387c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218357.92/warc/e1435c18d6f24f7f88c1d5f627decd6ed01927bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/1045543f0716801a8ba9ab027beebbda84c7744c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/28e5fc52e6061f15b1fdcfb79f58fb07570f5eb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/30e898764628f66eb358676c928a2e435741d861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/35b40173ba9287207b5c995a8244681532fde913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/6b0369a772e58cb6cddda68b9df0fbf347c932bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218391.84/warc/93566f38a72cee1da7ee1692fbe6ec73bbb39959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/52263f8068bad5389e6ec12680f94a26e250d7b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/55e0d6446b220e60b3840d90fa06209cf085e393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/8c1f8b7c94d47a91d94f9df01ed56be0eb721962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/9fc09644cda0c7fa72a3363d22ecb332c58fdad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/c94f22cdee11d0f742d11a5a09355c5e89d78098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/d3377fe6bfdc78524bcd99f11aa2953c04a221ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221218899.88/warc/fe4136a9fee113611152905e0c496f8f34650fcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/02eb7ee2b6546e7e3bfdfbce0a7dd45119a8e0e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/04777b2be1d3ee2764e562c246c85dfd3dca9454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/35c6a0af14bc1a2c9ee982187ad03a755fbc0301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/939513497c6c207d3c364f2a8da819ebb5ac1b84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/9ff4402fa02f46b8525909d5f035abe99e0f6d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/a8a6f0ae3ea50b5cc9d1e7076f7843c106171622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219109.94/warc/d1c6bef67a250274b9f6e8aa82cdf4b6baf4a858.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/1ea8b1cf2b44952862b5bedbf0299931199845d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/522bbcc99ba3d46995e0bd6b6a0f0f1accdb4c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/918b108738c54d8a4d7adefaf0e062b2efa0eb96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/c168f694b434f4c9b8834674b0206e5a696da6d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/d6fbd492e4fe0653548d89955dd4158a0f856005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219197.89/warc/ea0faaa9321c6852174069135abc61fc039b83a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/2945cd6c4b1cbc546c03383458051d03362da218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/2b8112eb2384b84da5c107fed01289af864cf42e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/59ae8dc9413e4f4306720d61c7f5ed456afcec19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/935943e1897d4ced5e310d90acd8cd79f9a06c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/da4e9624d3e5a0db7e68d6a572735d1ef3ef1c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219242.93/warc/e5d752324ac78a21a4d3577a262184f6fd256d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/3c770ed1de02623693954de0d88bfda4414cb218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/3e0d94502e80065b849079cedf77c984abc30b2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/495dc314418989d7ba0dc8ff259551abf1cbecf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/a94c21df8f66c3b7a68577a566479f9f6f6ab437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/b109a0dd0494fa290cca64e61cfbbc7d48afccdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219469.90/warc/f6cb02d61ce8cf0292408f92a18a7be2c1b77a15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/0245fed9a9b5692a0c8ba722168c82dd6b7cebb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/0e5f45b7059d66d368bd09110e491b786acbacb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/5b7a79767fdba7134cf3c7a322cef28a1b40b799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/814211f64fc9851564adb858f4d9a3e10b420817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/9f0b8d2a01b1dc49a501edb982697ef4379ae8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/c383b5699e85203d770e01aa7c9740a178566df5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219495.97/warc/da5826e880417e1dca747a642bc6e5bdc31df9d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/13e9aef79a3b4e47345db107d6ffb5f9eb47bf27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/2dda9c44bc4499baa53c68af6823295844b4d2c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/3b317afbcd25f2cf1be9b84ee6ba65a24ee926d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/55008443e7e872e8091b21528cbb0918b81ff886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/60c44fce7b8c69936f14b9d152bb761a4ed6b3f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/b20c4cd3b3b9bca26acd2635867ff0a2c0dea4c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/e4f901dd398b9da800ccb2280fbb148db14bdf77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1534221219692.98/warc/ecab17b84ef4c8d77085f1421c36101ef5bbdd8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1b80f680433374ab54ffe64288c519f89135cc12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1c286e9a98ad5a4463b4b06beb0d053524dfe353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/1c81404d2ff122ecdec55b755856f5b3d229cd6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/21dceb79c6a4072692919e4de16a625b4dfba444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/29af643f9c664e60f84b9cedb26904e224521a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/31e64dd39a87b672b2eff31fcb8e64e01f83c2b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/320e1d5db6214f298cce7fea910c672a8d45d092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/32a81a909b33e6db38b5da283472e80e1fbd834c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/3386410affd360f79cf42d7bf9d76906496548e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/33f23f5ca2eeaad59f86213ac83f176266e4fe5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/3565139af0ef633dad63abeabb6a69de9507e1ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/36273cab53ebbf062867fa2f71ef892c4b676553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/369af29bee40ba2bed686b0ccbc2b5fdfb23bdb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/39aa1b0bb08d5527d01afe5d1df3230693c974d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/415b5b0064038d39ec4672448a6332395bb59814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/4639d9616d2f5ca29d1f0167604bb417d65e0120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/4f6fb487e7c5eb223a0fe2f5ef9c2f1ec1cdb49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/50da67177391718a0f137e9eb7621dc32fdea3a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/5208236235625f67812a3644ac1a33eb7169a61c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/53d2db6d2e8d92e8081f1b014a77e11258cb959e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/54db144f9ff42a1a6e9c8969d41de0f035b5cc0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/584277145f2fc25748bb5a374c5cf57bf68d1913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/59975c0a6e15775eae24c6e4e87a4e0f9d46b039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/5b56cc2285d54ce02856dee595bc4d1b84b76633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/5eb2dfc127212ab4b0edfe105c609e14e2e111d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/60a1df6bfe4a7ce3ebd7b899b1d3f2a6876e87a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/656d6b9dcfafb5bdc8f20b0c65ea64f08a4106a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/6fe16d9765bc17d1c74c4bf2308ca8762a034900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/7677d127c09fc08c19739e4b848967bda02dfcaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/7756056dce492bdc2cec9354f0f8f8e398d98ed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/7d680fc60515c0328fbf1d7fd7eb7053edff1745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/84e381e6e8d577ba686d041bbcfaf02784518980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/853a81b3835ff60fa55d3b47e5247fa89993458c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/85855961322e63e566d9b485d3afdf1fc1959c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/8e82fa1b5ad3317ed08fe4a78a0d00be881cbf36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/918bc1a84eb8e0da66c275a7de2d4b7e478041bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/96d715c1bdbc4f6b9f340688d8318dc34d6724c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/9f18e205a7138d66cf8faeb2428fe1fce9f11232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/a07eb0c72d644193b5ae717778b9866c8e52de6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/a851e6c67fafa6fc780e1a08db675766a18a8756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/a94cf23591fb0f860f29d1d722662dfbaef44c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/b3e9e2bdebdf0d1e119aac69bd3d0a6eefd59e14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/b812197b5f92c9873a9cbdce5a7949df4933aa27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/b9a65855647d616b95619f1300d9c4de7b440ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/b9cb8ede256da9e9e4cc812ffdd88524d943099f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/bc5d0a099a90af028a49a6e708b5699772d605a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/bef4163ff0bb6809c6d54020a5c89d368a094346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/bf1a5cd0353c34139d2875fd217d583c83933d88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/bfd653f05c1f250afefcbf8cd8c0e89aa14ddef6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/c55fc6960a6268e02503d86e62c5cb6b17a5ebe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/c833756daae1cd46fa09c64ef9742f153d162ce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/c90b58c7f019f10106ec1275f92c5ae2abf3ea25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/cc426936f426919dd2490792dc0869403dffdeb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/cc6dde1f97f55a486c977ac12605a9fe106e63ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/cd3feaf6b2997a388bec48a76c454b0749277e24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/cef4860ed0d50368fc511a76a9dd58ca6aa54a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/d37acbcbad840f42a320b0cff8e6bfa81dfd0e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/d4e333056d970fc95a21fe7e1a63af8876775a8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/d8744e39e334c216ce9f82de09f84fab17b1da54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/d906033e09f39145d99fcf06c49733d89e57abff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/dcfe289518132476ac1676a877eae8a6de0255f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/dd67508de8edb56f168825f56498324783b21606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e132abf3aec261783c351f15f563d30d9049b183.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e3e78a5781529f5ce0551ace25c568ea2de27c63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e78c324730609aea98d1616c43adb44afc60cfba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e7ad174f9fc19f4a8df8ef7d8d0e39ef6e82f0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e8511560751c0231c2216134a8be085fa6645b4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e8d18f03f17f3d30a1c0bf3a1d287c023555ad71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/e8e5812b6a3549011c1d203c99dfaa59e2da4df0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/f408a46ab91a230dae4471ecf25c1739e926a6f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/f71b89df6607c89e33ee575e8edf657b8a4a3970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/f92ea1625c842a397b9f9e1fe50ef8dd8dd3f5df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/f9ea765edc793d3a2f4f64b8bd3db3fa5b7041b9.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-34 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-34/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-34/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2018-47.yaml b/configs/cccc/mixer/CC-MAIN-2018-47.yaml new file mode 100644 index 00000000..1c1cbed3 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2018-47.yaml @@ -0,0 +1,669 @@ + +streams: + - name: cccc-CC-MAIN-2018-47 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/0c2cda2f53bbcc3daf91ef8a0d3caf8545eb7b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/0f9e7db85513bf476993a5ae44ba8d9c495cba92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/115ab5c9463c991ff1b4bec4a446a5b52f75a194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/122d3e400632716202bee22b49834eb65f0f5b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/140fa24dbbde3a1c24688229eb96f7e36a3c0d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/14f133dd848135e74ef335d21f3c08c42e23d778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741016.16/warc/094f06e908246db2475088014d6a2a8ffa3cc9b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741016.16/warc/534bffd236ec4fc2b6ff315ea2d26f61452ce846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741016.16/warc/8bc30046aa0375a11da4edeb8616fcad9ea046ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741016.16/warc/dd4313f63a291a42026dc494afb3ac3b8c2c5a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741016.16/warc/ede5bfaf18b830405d1e2215379ce7d9d96cef7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741087.23/warc/0455e428776681753b8fab83f502454f482be9d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741087.23/warc/40344d331885b5b5069c409ff2e00e31fb6d0e7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741087.23/warc/9413282fed941aa9cdefc66f849b7bb1dca9e803.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741087.23/warc/e50a855d3155c9b3455cb9213f6417220db206b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741087.23/warc/f2a3ed122197b67b11c2f469c8df9fae503a921d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741151.56/warc/607125867388c60960326db783e87ee1df4ab5e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741151.56/warc/76ec07d9e24fcca8ddc0301dd5598331eab01e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741151.56/warc/7c4069dd3f226ae191b86ff9ebb7ac725e8dc771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741151.56/warc/9135cefbaf57a5c0ee4ecf88a0e8551b216fba44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741176.4/warc/5b98d9478f273fc0ae934a8336aa5298fc7ecaad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741176.4/warc/8185fce0c13c6d63fa688b93d3f46dc90dcef954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741176.4/warc/cb5547c4cd4fb0aecdc55745b6d39c79e93eac5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741176.4/warc/e215a1643ea8bd37ad81d342c3db3bf840f9d3b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741176.4/warc/e70457ab27eee8c1ed61fcf479e894070b4bc9f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741192.34/warc/069950d81cc9ceeeeea81fa7bcf8511dde646f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741192.34/warc/4b9ab9b0c0a67faeff25bdc869f31339903c1195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741192.34/warc/84b4246176c7b4ee5167f8b3dbc56e16bb07ab7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741192.34/warc/971f208c76d25c0f7f3cbc24281a8fea4505e9e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741192.34/warc/9bbd042bd0079d8798547e32d47e3b482db40292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741219.9/warc/2416e1ce9df2a902230235c0fa1f87a770d9d838.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741219.9/warc/26ed6f9c82ec6338023a08ec2054504feaea8baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741219.9/warc/8bd890948ec6f85d4bba8b3ce2edb00a19dce080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741219.9/warc/8f7474362bc21ddd10a7a06208b4dc7adae1e706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741219.9/warc/baf6615e6abd7593189853ee9109a6112dfe0adb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741294.2/warc/58b613871ff281335ddd5a207fb2412290e80f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741294.2/warc/738c7f2156bb8cbd65d359711897b41c9e6f5a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741294.2/warc/8fbd812bea663ec9aac02e3a6f84485b3421b898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741294.2/warc/a73c806fcf1c653c46ea338e88af53d6212962b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741294.2/warc/c28c26f918a84a684f297875eba068351167c63b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741324.15/warc/0f44464d9615ea254b0500b9946890037bd0c09b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741324.15/warc/268f0bd42c3218515711b13c1a885fdefa9c2c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741324.15/warc/516c860b9773aafbd53c32ac0677488dfd67b802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741324.15/warc/d615e3f0337112d2daf8fa9962e4320c8ff2220d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741340.11/warc/0803bd4e8eaed22309ad353484e19ae54ca8cfb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741340.11/warc/2a0489774fc2e4f6833cadfe94430a92d0aeec7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741340.11/warc/37a8540ef1b98b04adce233bc01436cdcf373505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741340.11/warc/3f7bd19e785fb656b078b48cb89ff63e66d5e947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741340.11/warc/8e4a3fa147cb6327c858c17082360b78bed782e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741491.47/warc/378825c0e32e95b626d1162b51154a1cce82c77b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741491.47/warc/4dda19e593bd0c844467b92327af708cc5cdc768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741491.47/warc/6f3e08d2a1e61f88923e2b045d6624aeaa0a6d22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741491.47/warc/933345edd5b186f8930a68c06fc695ff25cc8a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741491.47/warc/986b5017c6f9ae638f7526e8bb3fa84111de4542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741510.42/warc/0ee8f15e267edec97bc5f40413b894fe781daf36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741510.42/warc/4838fd39425411211ecb0d2369d8db2a2600c0a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741510.42/warc/941ff111d78f0cae582d999d041462cd57cfbfa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741510.42/warc/c425833bc29cdf272a9bf70a8e7fbe5fe1d478af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741510.42/warc/c690dab09d06625b31522d1c2b1fe95774e3b6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741569.29/warc/2eaf74c2774a391186ce68bb4bbca1e122c8af01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741569.29/warc/86dae36c0593480d0618ec77e097438578aa7e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741569.29/warc/ed68a0e7f67bbd1d72f3c70abc9976214fd12068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741569.29/warc/ef37b719df59162f1b0c3237b17ce92f4e5efbd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741569.29/warc/f2ebee32dae1d655c5764221dc5ab6424f8431f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741578.24/warc/52cc96b4f165bc71f8fad8dec852945ce8bf0b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741578.24/warc/5f3132eb42fd0fe12554285ca8fd9f1f496510ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741578.24/warc/733721de72890bf2168795b096de208928a18c6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741578.24/warc/86564a97d581fb98867abbd26db55d06da1eb325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741628.8/warc/5da644c912f57da92187469afa028da5621291bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741628.8/warc/668f4256aba02ccf4738af388f787506b547d5a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741628.8/warc/ac17f08b214edc7c373d5d319f5a29ed11ba07c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741628.8/warc/b255b1f716368bd52aeadadbe0c422d50d17ee47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741628.8/warc/e064ce281a394a1aaaf046a87e312763f004fa9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741660.40/warc/2c81151097a6186076aade5ca119a68bb8305d2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741660.40/warc/532bf423d9bd732e6fe8d9d7ba388939e74dac1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741660.40/warc/c607b4fefbf369adc8dcc64b007401624b05fdcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741660.40/warc/e209984140d864b31914b57fc1148f38b9d87ef5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741660.40/warc/e92937f5d569d34175c295e41302889fffb5e039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741764.35/warc/3b1955ab4dca9709e5a803fd3f8766ce743b3b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741764.35/warc/6b9725b978fa58d51b26930b08b45150077f56a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741764.35/warc/7984768d8453eead2777989fd286ea3f01d72228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741764.35/warc/d8ce899ab23a656163d1f833bf02f7c0df740d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741764.35/warc/f85a68a225a53b417c04ae6c2941d3039b1462f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741979.10/warc/1c8aadd4240347011e762cf42fcc4d3878aa2519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741979.10/warc/222a82db36e4fb635e3441c8b9e281503de993b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741979.10/warc/5007181acccb340443f0f1b0b4ae94de043880ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741979.10/warc/da5586d9c7ed847c54f80e6fb42a5c498f519a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039741979.10/warc/edb5d381e684a3a7a6eb580c01ecbeb54b236b3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742020.26/warc/79fa5fb6a02e50a1894d1f84cf4d4d211fb10e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742020.26/warc/a0bf872d0394df2f6e192fe7d75c00317c5e2ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742020.26/warc/be2254553f1236e478c35d5e641c0d29c50a6e66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742020.26/warc/dfcace55908835f220be60c3f0cd6f0640e01426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742117.38/warc/128de272125fe9422e517bb2cb1b83ce2bcfe95e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742117.38/warc/2546e7344fb0053efbe56e7cd75528205c61b5a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742117.38/warc/898ee6a581fe9dad691c7ad2fb3e8f2d60eccece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742117.38/warc/908ddcf1de1130dbbb45cd7d46d5e1a62de21fe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742117.38/warc/ae7d4dc3a29364499a32e9b0b5e3f9cb497e1b10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742253.21/warc/002e40307a370ebe759f6c98bf3222dab69e92db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742253.21/warc/0775922e7ef8586592f4da2ebbdf5031791231af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742253.21/warc/8a85f75e948bfa4a9b15cfcde052c609e2178b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742253.21/warc/bdc6cf7386aeeab52adab313f6b54920a64c55f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742253.21/warc/f321dd0078a26e954e0b366ae79eb1859cd5c4f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742263.28/warc/029a8516cac46638474df51d5c28947ab3be946a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742263.28/warc/19bd7d6c2ae4ffac33f2a0c5bf73e41b1e4644ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742263.28/warc/1eb9d1932f2da12d076c499360771a9396f2d644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742263.28/warc/70bf1f9e6d7ebfdf0dd348ae86418d740505b766.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742263.28/warc/a99d608361cea6263e9cfbe1d8c28dcae58a032e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742316.5/warc/425ca768dbb74dfae57092c96f39d82e22381312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742316.5/warc/5785a3bd8a16849d59698800179ab6b13577058b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742316.5/warc/8f0319fa36ccfa6daec74e902f884083685bfa7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742316.5/warc/b153c88d7ba344e3de7d243a5f6cfd9d18b33b22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742316.5/warc/c8fb80cb2d5bbd8eca252732ff9080bff6712e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742322.51/warc/631e327f4645271c880ff86361c7a4dc85683816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742322.51/warc/9326d343525761618c3e90abead5358d58e81f16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742322.51/warc/afa3e0cec8a89bd39f1227b18e8fb1261a1dd61d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742322.51/warc/eb742333b1f8f943e783a6bac8a121e121923dde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742338.13/warc/5a621f3a54db30ddfd4b94c4398cec21c2602085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742338.13/warc/5e15433070d18982474d9aaf55c7620012604d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742338.13/warc/7dbd46b50f1b27cb36fa3283f983caa23bfbfcfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742338.13/warc/7dc0e3d5a4adcbafa70b3f771190351bd02bc84d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742338.13/warc/c2e359de4bd6d1e649fa2d288760c81c2cae37ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742483.3/warc/154ce9ccfe94225004b80b3e3e2b351b84330c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742483.3/warc/7f0a8f3b53fa004b8e5eb3a0c181e0d1f391dfc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742483.3/warc/aa683faaee389e451171435f77416a3df36c2cf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742483.3/warc/e391cdb820d52ec06e98c191ecc11cc5f6a95f48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742483.3/warc/e3b5dbfcee0df56f6f86c3a8527c569e11c37354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742567.46/warc/06b276eae5c704fc7cd53f02c0e7376638406f5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742567.46/warc/11b5f3cb6e7b0d7b625341cb2f5af2a6855e054f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742567.46/warc/15cf0fcee9cfdb9f3091c0d4d4453e9cbf21b5b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742567.46/warc/229128ad306fa1da7100a9c14c379810c44595bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742567.46/warc/30b5a024d7fc90c41504dbaac884aa40ae9b9822.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742569.45/warc/2ebe419e6d06f5fa74f51cfcd9add5a68a375e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742569.45/warc/59e7efcdf22c69ff4adbd47719a4980553ab63ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742569.45/warc/5f456b64064dfbd245322c2721474d74717f47b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742569.45/warc/bebe98422d0b85924b3011a649044d96eb1a62ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742569.45/warc/de6ed5cbdf4796780d8cf700f2a40443b7c14e54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742666.36/warc/7d9a670198e56fff980c0a3bb7819e3e79a8f3b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742666.36/warc/94e658bc6199d2b0c37fc24dfebf5a3c5ef4672c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742666.36/warc/96d2f5e3e5d2af039c190fa5efe3c9e08a2a5655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742666.36/warc/cc8772530a7c29543e1a4ac35f0cc1964f0bf245.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742685.33/warc/067879005e23a778b0dec38fefcbc029cdf3ea21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742685.33/warc/4bf1b84108b754bc72c324e60b0f5fb813c67abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742685.33/warc/66a4a3edbe6ef4d911e2a59d3a1b2c5a85b75e69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742685.33/warc/72111608c02a2f7a1ca4eb857d0832e2bbfd732f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742685.33/warc/ff68b3a24a10bf57bfaeff55ee8f31dd2ddbca2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742779.14/warc/219889b366d00f1982d8a72a1888371e9c8021e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742779.14/warc/5103dbf5ba24268ab82ac391909e1b8ba7b3c6aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742779.14/warc/b6244f57b6d29f5b055210ffdac62f1453164ef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742779.14/warc/bff75fb2d6e8409750e5f4d1b34d56bd272778c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742779.14/warc/dae9bf2dd7768779857188d43cc293d4ad0ca889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742793.19/warc/75bc51847392191ac37a317d8dc6b8b4420b0160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742793.19/warc/77c9b50bd61f3b0867237916ff6c8316ddee8fc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742793.19/warc/8aa6532359c96126853c4f6afc44008afd57685d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742793.19/warc/c821999b42d08ea45ca7d18c09db0462ef132d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742793.19/warc/ec4b1694ad0fa7db637dc05c213048861805c7c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742906.49/warc/2ac6d11a82925213521b98bd34be176ce8133bad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742906.49/warc/2afa6c70dfe312293d8c6c1edd9c084098f969fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742906.49/warc/2f889f341d494468d9e59835e88e00210f51833d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742906.49/warc/836d6485d36318cfb65bbdd70f484a32e31a12c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742906.49/warc/9eb3839213efcdef0b29e78d65f0a83598153a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742937.37/warc/4740dff9d0c63b7b92034322e3488393695d4697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742937.37/warc/49ea6e81e1cec55fd526b1ad11efd0148e779809.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742937.37/warc/d1c935f5eb5d14ceedb8c2c7fddf19f10bb8a039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742937.37/warc/f008e5a5fd5733fb280e4d119b671b22b4ac66b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742963.17/warc/457ea19b1ed86d3d73b93154cc77c3f2d6cd97eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742963.17/warc/6da5dac33670467806ea7f704e3895db7f87578d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742963.17/warc/7a970ddd50689ad10c3274e7bbb6988884854119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742963.17/warc/93a45e4bcfae92dd94d046cb985c7a0dc3808001.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742963.17/warc/ad063d07e13eadb7240fb12509402c1b749ce947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742968.18/warc/0ea2383f9f88ee39a699510eef7b0669c5c55a41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742968.18/warc/4d57ea361d96d42f99b15cfcff370161e58b45f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742968.18/warc/744c62ea67b99554a28ffd282941a6f86ea1f4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742968.18/warc/bcaf1bce9aafb1647323b0c72ac16f357f27fd6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742968.18/warc/da4e599d3ac9ded8e78096abbc52cfbfa10a3167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742970.12/warc/94e301a5288d9e5e8f21ae9a7f0171e5b3664b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742970.12/warc/ae7df2bc0a95d8c5e55b864370bf2e1ff134f548.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742970.12/warc/bb857e937de33e1ce063d5121e15a8092c408d70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742970.12/warc/de6834fa779f4a1ce9b95496fec9c6d46f7ccbc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742970.12/warc/f77c79ec6cb596ab685c4041486d231e0754a310.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742978.60/warc/0514670c4b7d56e4d04804f8aa01f9fec29db64f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742978.60/warc/2191a4cceefe05d2244de03261f627c2b0a051e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742978.60/warc/346f1f8d4ebc362e466a6c7f458452478a912746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742978.60/warc/c633e08986b0fcbd79821a249a4822a57e1dc9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742978.60/warc/dd0de8b2b54508e30d9c84c6f1dfcc1b21a69e12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742981.53/warc/19b3c4b740148a474c8ad65af194bafd7ed22913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742981.53/warc/1b04328b1ed8513d67282ceff958465cbeac1761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742981.53/warc/47eee44468313bed1136728d83c340432b7338ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039742981.53/warc/a53c1c4ca0f63e79f98cf4716f5db19e44750b49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743007.0/warc/1eee1821c6bee258ea67520ab1c6243502b10ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743007.0/warc/3bbce5889a58e39396bf1d44fd35b57619f60bd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743007.0/warc/7567d3483db6501976496d2487df181a49e763e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743007.0/warc/97d83722dc5bd44d26530bc62ab6e3cf3ace8f38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743007.0/warc/aa78356f5f17283f685c50b217be479d8a3c9d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743011.30/warc/0a9a31156ddcb2a2d7f627da53ca5606e27d5a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743011.30/warc/23293db5660329dcceaad2aaa5bf1f7231f4dc1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743011.30/warc/3f6510c128c58ce9ee930531db80f843691f4438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743011.30/warc/6eac91a9d7c0fc5cfc8593ac6d3a841351882d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743011.30/warc/d4308fc513f12da83e1a68cb3656d5d166374718.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743046.43/warc/7c6d86c2f882d2082b6fa8844131a569c0fa8db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743046.43/warc/d64e93e1c4b524693252dee3ca6805ae3e4c6f22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743046.43/warc/de92b8c629ab37de3690439c985cb4f478d9c096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743046.43/warc/f35aae1fa02ef9a6b10be83d35a7b367b98cf15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743046.43/warc/fa7fdb9a1e21a599979d688d79a65802d8d8310a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743105.25/warc/027562e004ed08a982a3958709760b369eea9a59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743105.25/warc/02d2fb5ee7881a3abbbe9e3b6fc1502d374f010e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743105.25/warc/7907988ffa1c69a8eb044ca633813fc08cb3c17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743105.25/warc/f300d3bf35538ccb6b9147d3f12315118076a935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743105.25/warc/ff491193508d312b0cbc68ddcff0a9d7291247cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743110.55/warc/0910891379f8288e05f7b1002091e546b22577ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743110.55/warc/1e47cfd3875ddce0012721b4aa86d6c322a0a3c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743110.55/warc/3911398a2c813c62518c3f2af741fdf3108f9c6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743110.55/warc/ba3c813c660b39ca5c8d2b2d8af1b49aea7ebb84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743184.39/warc/003a7143197808982eb557e50951fbe8244fc2ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743184.39/warc/4d75afe39fe8d7487ffc5711ac572782e7d939b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743184.39/warc/8c61e286fdd5fd3cb10cf4461cab73178c7e42f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743184.39/warc/b4f779c5520ce08e737cac3fd92abb10eff5fcfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743184.39/warc/fb3be3a6aa7b29332af74ca3b4f75faaae665f31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743216.58/warc/18c79fcb096e3507b2a02c421516036c5e26e992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743216.58/warc/3ac0e3baaebd84cda552a3684aa62a311fd4695b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743216.58/warc/5358408b6c5406d4e5b8ce25ec9e91c45afb456a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743216.58/warc/ab5da750e574792269da02bed941efdf4d528d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743216.58/warc/bde743a33ea683f9ab136322f2fe8b8fae092dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743247.22/warc/1249b704a2a32cc6da6d45521024ee06dbb49ce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743247.22/warc/5dbea483179d2c05cf51f7a4330a0ce6b1c7e0c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743247.22/warc/863904c7111cc98b276c0a8ebaf9ed139883a90b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743247.22/warc/bdd38484f71e70ec1e940353ea073c2ee9d70023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743247.22/warc/eaaf70e8bef04709ffd705c8564800605eff81ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743248.7/warc/724ec54ea2e47d8eb0d5f16fd885fff1b3524589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743248.7/warc/ac3222766d9b4d622f9e9542738c029c4c7790fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743248.7/warc/b6b26965576f8da6be0c57981d404c797ad1803e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743248.7/warc/eddf314a459782cc9478b2c55351b8dc2d03bef4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743248.7/warc/f7b637c98b088e82c65254388eb8f19c5a18d484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743282.69/warc/4d6e05f70c2eff82ece7be26a7b10350d5eb450b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743282.69/warc/6cdde59e89874181184882f0a866148cf7bd6767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743282.69/warc/81be99f4a22c6adbf8d2f86cfae26db80bfcd78b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743282.69/warc/c5d013e54122ec2fd8968f025693941e6e251bf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743294.62/warc/4526be01cb1590c5049fd33acbba1ef48a08c24e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743294.62/warc/b5c9c9f80209dc401de4efcd99d5b1262afea1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743294.62/warc/b81e05c99c7db8413daa0a65f47c4bb3e19ffbc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743294.62/warc/b93133f783cc0b3cc4f484a12cbd2218d6fbe16d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743294.62/warc/d5fe26bb2a7e9b8ef145a52fbf92bea396deb6d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743351.61/warc/1605eb814aba8ab8dba377fa416bf7976c16936b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743351.61/warc/6082ea5227ca38ddbecaef34012ec07fc76ffc51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743351.61/warc/84894700f7b164b230be7c049dc9fbb64f330ba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743351.61/warc/8d0ad1b92ceefede2a9f33951d9bfa873cb1e6dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743351.61/warc/f01e51216289a041d120f0b003ee2ef082aa6cec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743353.54/warc/3e3ab57d58d141ae7c25a30f34772628e6c26ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743353.54/warc/6f1247fe64b082ed22916936edc4374eff347b8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743353.54/warc/7c61d36b3600ddeb8a74ad06ddb5d09f3941f7c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743353.54/warc/8a8ac1ddeffb939eaa40bce0368dffad843aa278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743353.54/warc/8f46fd434329c2fa04f4ef5eeba43469e71abd51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743521.59/warc/340ee6a5e630483f318c5f935767f783c604f8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743521.59/warc/5740ca5792f44acc1fcf9fa70f039640238c4dc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743521.59/warc/6f994b1526c44583235285c36d86aa79d0eeced8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743521.59/warc/e773da18d5efcc77a4c0eda50781066b13fc6bc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743521.59/warc/e7f4e0b0070bd08328cc38c6707ac6667e3cf4bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743714.57/warc/35c71c589ffee4a41f0956b44e8b0028b42046bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743714.57/warc/37b95cc81af4b76bd7c78985b55e4820d3c8692d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743714.57/warc/78170c32056be471e8040f7833d50ddd7f1d4eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743714.57/warc/c2b1488cc221a1506602dedeb21c983fa1a112d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743717.31/warc/23829ffd66fba2038465781239ae3389bf49ae40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743717.31/warc/32ad31171297fb0d92af44e6943a2a38ad085757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743717.31/warc/52ea2f5246f5cde43884eb2aaa5eb7a4632bd5d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743717.31/warc/de4b49f268862135fd9d737baa253f7900c170a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743717.31/warc/fe7e4741f736244106f18ed448e594e146a1609c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743732.41/warc/3219d0a1e671c5952507ee111664de19f81ee918.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743732.41/warc/71055784007a41f38e0517f551090a4d9d55edf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743732.41/warc/805ed79d89a3884f2eee7776507e458aa6bd0279.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743732.41/warc/aa98d7430e796805d49b035afcbfa82b132d8065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743732.41/warc/dedf225d64663c1fa0edf405bdad937f79415452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743854.48/warc/16c5e6d03df586e225ed9460b13a5c4d6242fb5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743854.48/warc/80720a3da70e92552f149c34b19388b91fc2383f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743854.48/warc/8b404ecbdd567768ee7964a59441ae1a349852fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743854.48/warc/aaa1074d6397e68d0a565d8f6325c837e8cb9a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743854.48/warc/f26bfc0b3e64635e65acefb9943bfdd1af1f9b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743913.6/warc/135e03eeaccf2fb8cca054721fb3cfa8d3af24a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743913.6/warc/2da50fac2a1ddd78b16faa4499a8e841845539be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743913.6/warc/3ec6df17631b8345d2946f7074d4399ca861b8e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743913.6/warc/ba093320817c6d3ab7b842418ff43d22e55b2425.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743913.6/warc/c4cf9f8b8495a1cfb9e2bd85fb8dbaba9837f83c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743960.44/warc/2be5c67278d0c8f4b3d8c4842bce3973601b8af1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743960.44/warc/37386560590a24c974469e22b304f2428aff7fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743960.44/warc/9b39f7ef037ba62add809054ca6ffea823bad209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743960.44/warc/bfc67857898a22a2c2f41c8c62ae64b164765593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743963.32/warc/93f6e77adccb690ce3dd6b448b37aa4f96d5f72b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743963.32/warc/9ab2fb11c663d0a0d400a1119a681f7fcb4356d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743963.32/warc/c4e99bdb623184613657ced37db413cfaf0d3aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743963.32/warc/ed399bad9e8501b6c211679e45ae952136acc102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743963.32/warc/f1ecc62d811e17a360ecc49abbe429790e6afcc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743968.63/warc/2052f8e3b0d3359d1f257623de9412a5f0e273c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743968.63/warc/431855f609ce5b1759eb4bdd8979cebe7aaaf8aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743968.63/warc/6eea3d2bd7561cdb86a66e67ee1010ca83e6838f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743968.63/warc/a8f04e2c684d5e29e5f53b9d4a0ebd7823816590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039743968.63/warc/af7f6a3f8555404c744bb9b40d9d521884b14e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744320.70/warc/098149a437346bbef171afd88923f8052c90c936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744320.70/warc/2236986096e2aa3628b23c2480c51fa0c67dbeef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744320.70/warc/5278ecca39b4ef368605bd0a18923e5660b4a26d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744320.70/warc/a2659aeecb7b1dc3b7c32ddf07a7733f811cae8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744320.70/warc/f3f0691d6e0b7b4480fc2bc1f60ae382e3fb4d23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744348.50/warc/2cf689fece804bfd41da8e66c7d4a71227fa8b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744348.50/warc/5c2f274440108508afacd9d70fdc0b94573b7337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744348.50/warc/7ecf703503966b2a1fb73a5d3161f2bf4deab846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744348.50/warc/9092d2506dd7c3a0147be606b544ca457ca412dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744348.50/warc/e0a9da4acd93e4a7b5c84a8375f0f0ab7bd70770.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744368.74/warc/4498476c297645e2672d6df5f34db215cc02c62f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744368.74/warc/455aa00521245e54b984fb14830b03c982f9dc57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744368.74/warc/5f73a49541a2880fa65595c02413fff432041a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744368.74/warc/e5eb73206028a5752035b8b77044e9fae40e4d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744381.73/warc/780d0fcfd96bdb3627b19a732ed46b94e43fefaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744381.73/warc/af45c6bf859c8f16aea9bf1129980648dd473d11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744381.73/warc/afede36df4c81cb27c0905e1945a5fe788c662b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744381.73/warc/b3d17f202405d3fad21b313682b7cd5986bc1d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744381.73/warc/d618317dc9074443d896b3f52e48570fd1ee2bda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744513.64/warc/16746278557a6be95c5f5b84d975733f7dde06dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744513.64/warc/3ce120fde4734a07f0f4ea9e0f97aa6e1c9b8eae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744513.64/warc/59956cb20cf75a68f9f43808173bef22d13f936c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744513.64/warc/8cc242d207ba4ff15e2cb51b8a0b8adad8ef1277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744513.64/warc/d997a23d7023237761172a8ffcd5d8bb05bc73a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744561.78/warc/00a103446b8980f06ab3b4e87af5c63143d7816c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744561.78/warc/0701140b2331be4c6cb96954a192eb10617aaef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744561.78/warc/9f3263e40af4ecb2005045ec1e3d05afe141fdd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744561.78/warc/b067a47ad4e1eaed15ab36b172ad60f5d0b28253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744561.78/warc/f9951d2f66fc3cd96436f190d8fe9cc2df2266ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744649.77/warc/78b5ef43597d4d4c2c6f79e4c8cdd26da779dc7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744649.77/warc/ac495cce47f3cc6173a18fcc83007a7946fc3a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744649.77/warc/badc6bfde173b8ef0352946d6b828d0dfda2f9b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744649.77/warc/d19759ee00216c4986a9b1de0d0a58054b98f463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744649.77/warc/d38a82e7526ccc0b9d93a41a2924695c8b544b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744750.80/warc/1368c00fa77f46720c787b4904942871d3e33d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744750.80/warc/616551d25b5486dc428f25fd30fef197968aed60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744750.80/warc/6811b086b505fc8f0404ea0ba1f8ae6ce4c7b8c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744750.80/warc/916f1dad031a3e647df2de25f7970c166e9fba94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744803.68/warc/0d72db386f027bd4d4f6c78346f348b68c52be93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744803.68/warc/68b507d875d938724bd39bf89e33b303ef38bb10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744803.68/warc/a3105d9c048fce90a6fa9323f2910bfef863c9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744803.68/warc/b845a3935a28051252015197bac359520de2715c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039744803.68/warc/c57d60e974cf423968e95222ff512c815b984df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745015.71/warc/09de253e2686039c9a17bb8712a6f002b5f9a001.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745015.71/warc/0b9a2d57386de445565e369e501667a8d5db86b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745015.71/warc/6f3daf18bab92580d596606e82cf6c1bd242bc31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745015.71/warc/a02bbbf5f67b7935b35feb8838fd5bc5802abbaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745015.71/warc/dbc836b4444298feb1513bde6afd9fc1bd802d89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745281.79/warc/3c451fb6d1d0db8c37079c709905613c6ec597ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745281.79/warc/473dbf1175984ee09582a580a0d199cd90769bdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745281.79/warc/96c8b37e7422e08c3f2bdd61a3c88bdad0c10497.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745281.79/warc/b971578118ebbbaf961037a0b4a4339fb701b257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745281.79/warc/f169301512bfc04bb5ddef11269eaae2a37aec67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745486.91/warc/35967e427216f9e38c421ed76eb939c829b41d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745486.91/warc/833fc713ff91c7cdb7a6a801f63f37da1ae8fc70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745486.91/warc/90359c186238d8c8be6772035ee5334cc9dc83a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745486.91/warc/a64533388dee5819a81b5066405bc2971f82eb75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745486.91/warc/e1b398aa9b4561d02117cc51c552c7d6daec6108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745522.86/warc/2eb8906f99fe367c1adc0bae0d9b6d29d74a0661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745522.86/warc/8bbdb301ec6b3eab73ef0e516a5c7a0b519a1f16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745522.86/warc/aaa20499a257afd960862d9a240b232580075867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745522.86/warc/af317884b7178247413f24c9d247e00e7e3b4d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745761.75/warc/2e732d3ffdba2bc797291fda6120e6979d64c3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745761.75/warc/8e14453e932a6093f9779e07b422048da65a02c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745761.75/warc/d121370f76a473a5dcfc4fb9875d1033cce5a1ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745761.75/warc/e9700ea0ee2788bd8166849bfc7cbd632a7440e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745761.75/warc/f0b8772206e445e3eed639a984665230583748e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745762.76/warc/1525b3f558c3c85e9fe85e9d3e1e332545590ce3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745762.76/warc/1e443f09048abb30ddefadbe59cfe8d42b9d24e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745762.76/warc/3da5f69bba29edddae0244a522409879684e90d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745762.76/warc/8dace3e941bb0b7259132fe4a4456e5c470b548e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745762.76/warc/8e2aafe9b3ec8f38d3f44ca7a656455db608fb4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745800.94/warc/3e6c6c567ab8cb8935a71f55b3abd329d7b6b687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745800.94/warc/70753a696da60ee77ab93d2a593c5ac752b442ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745800.94/warc/846834cf3362b8801acb1f71b9608b8393418def.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745800.94/warc/96d95ab78463a28ab4371ad6a357cb2068d8959a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039745800.94/warc/abe5104218e801789c5bb8377831d78eda256b28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746061.83/warc/44f26d5b530d9818437cf9544a33b0d41a3fa519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746061.83/warc/4d72711f1fed93dc381aaab5c052883d56c72450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746061.83/warc/806e6edd9d59343c2c6d3415dee6281c050ea468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746061.83/warc/bf6c10fd83186ad7176bb46bf71cd9d412254b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746061.83/warc/fafe6d5bb4680db58504a4437929d7b93eace764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746110.52/warc/29b533fcee86073b0a8028c2f8d2d5cc51906d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746110.52/warc/43d3b1db43557b4a02e2030f23a837a40d1f09a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746110.52/warc/938d5b8624091be060e7685ad895c28e34014bde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746110.52/warc/9f9e20c5bb87ff40d854498c581bf129c76b55ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746112.65/warc/252e58dc5193e1ea5ab77261a742edb440f8ebe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746112.65/warc/a3574e2949ee3256aacda93cfef3187a4f30c9bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746112.65/warc/b720d616b46ea35a16f1403a5540876c21126a2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746112.65/warc/b75223baea7f48f0d752f42f5421843b55ecb753.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746112.65/warc/d63c5e90b1ba6016c1487ba7842f008f3b52c8a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746171.27/warc/30b12e352cb9fd67a15964b3dcaafc6c4a41410a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746171.27/warc/57110200f7f4d0c59aecac3f8fe2bd361bb401f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746171.27/warc/8b0e869ccbcc74a1083953c5fa42f48adb08cf80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746171.27/warc/9db9fc4b3c9e3747b56a14cdba64a79202db83da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746171.27/warc/ae22a491aea864999ba2058dccb28d7fe93d3af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746205.96/warc/117e3e839cb6eb8e3ace400ec9b568a9b8cfe1ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746205.96/warc/58c34d4e08a0557494fef319e616573a7dad5586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746205.96/warc/a4144a1602b7114c651eb413328f0f4605c4f075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746205.96/warc/ce6c535c615f1a441c99422a23d8a0af7ea66fe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746205.96/warc/de367ed52ae724a1d7f1887685b99f6b41ac5d3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746227.72/warc/0e372d80ac1997ff486e5374695760eecee564ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746227.72/warc/8dbd4003b8118e4bbf208dec4cedf7a5d5d9c379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746227.72/warc/92ca41527ca4f6ef8ed0d44e1dff3e35af490013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746227.72/warc/a66b161dc88544d259c5cf27854036a38b129c75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746227.72/warc/e93ab9270364431f0c38b64282f5030cc080d19a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746301.92/warc/15113460e9f8bc3363828155c3cd9e513b6bf651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746301.92/warc/3436064bb3f0c1ab96f250b6772d31871e92b813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746301.92/warc/ff19c88e90afc3259493dfb3089a22e16513f744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746301.92/warc/ffae6a41a592b44aeb9db191c667dda7f7c6ea12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746386.1/warc/3a4b34f859750460f5d6d1ac864253941273a59a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746386.1/warc/3e1632e8bad00e841b0ca8535e4e2ffb2299ba46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746386.1/warc/5ec91ddc2ead72625181a181fa959faef5138f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746386.1/warc/d1461825757562b62eff42a07b80959970c63cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746386.1/warc/d14faab25fcf25c5d90408c523d8522392245646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746398.20/warc/4964171dac7bcb1a00505aac98dd4a53450f2d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746398.20/warc/5919fe6bca69728fb511d41262c054589a58d4db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746398.20/warc/5a009dabc1cbc753ec476743b1a761072e44d5ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746398.20/warc/b74b72396add0401b566d8318197cb9f6caac165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746398.20/warc/b88441fc62bf0be0223f80519622918379d21e5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746465.88/warc/1373f9cbee6bdf7f27e986e205c0cd67335a5b52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746465.88/warc/9c05540746708086d2666047a3f73966d8e4429c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746465.88/warc/a32b4905e6c92361a7094b8ced0615a7156d98dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746465.88/warc/af088fb8c5e4ecc00e4b81c87eeef33dfce9ab88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746465.88/warc/b11acbdac0c0cc07ee9ba1b820e2b810d086b619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746528.84/warc/29ad356eb4aba107f0664d3d68aec0fb502fc22a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746528.84/warc/4e435d4d413c14e31113057752463aae1e47887f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746528.84/warc/6f241efc30a6a69ff25152ab30bfab9fe7b64866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746528.84/warc/efd8e5f3497186c167de6e8df9ca09f7013087d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746528.84/warc/effeb5bf2c4225478bc068a123446f5847e48207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746639.67/warc/2f006de7f8d6c15fadaa67a675d6d502bf43b094.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746639.67/warc/da4079bd7cb0c790bf2ffaafcbc6abbcb8c9b0b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746639.67/warc/ead55650bbaddf94902ebe1c7944adf17172e167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746639.67/warc/ed331a874abef077c760a3b5fe7ded19c6c65e52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746800.89/warc/40dfc4e38a09f62c6ec16c92bd36817230e3e0bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746800.89/warc/4fb692b8de5d70423a410c0f3bf6fe4f7480f69f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746800.89/warc/658902d8095e4989484f2d5b6adc83f1a31ba7d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746800.89/warc/95f2feb1c71e804a58578d37fefd4b6740575c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746800.89/warc/f39bfd66b10291d8adf68d72a14ff81346d1dfac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746847.97/warc/2dd2f57344be830da76bbfe91f0ea6446b97e3fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746847.97/warc/53180238d69690f96889a14beefb2472532a09c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746847.97/warc/70c6e1c7606db2420f6169f57cf7757492a85c13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746847.97/warc/c8fa0a1cf45d3b66e037c01bcb172f4b84cb87fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746847.97/warc/f2fc122674a38c7e0bb7e69b4ca78223c0d1a798.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746926.93/warc/005ec3ea6388c09175d3cc2c50a1bc219eb64e43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746926.93/warc/176b5948b18f29e557d73c390c7659ceb4997c6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746926.93/warc/3eb7dff8743e5f3d304a5a001fac34e920ce37db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746926.93/warc/950cd7372687ffa25fb73036c6e0b85af8088456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039746926.93/warc/fa84b9610ea8cf49813fbcf2f2d9bc62c2ac9327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747024.85/warc/251fb21e3c504739d239b11282a1a0a6df62bc92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747024.85/warc/3717ee0593c6286957607269ac7a057976556460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747024.85/warc/4505882759b42a943a1c536e9e0db49c9d547ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747024.85/warc/b4b7639768b474281a42e2d8207240756c47c8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747024.85/warc/d97aebf314ccf83463b8251a61e565f1a4ba65b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747215.81/warc/342e7bc2198158fd269ffb4cc75420e07274568a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747215.81/warc/5ae7668704ebd8b04e0a0e121c631b2cb5cfc8a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747215.81/warc/6dc8e855c3d6578af0f0f68c2d25c754196dcb94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747215.81/warc/74d87836c62083a978861f1f57f58bba6f4de37f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747369.90/warc/0b767276804d73c96ec7961ac44f0622c7ecd46c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747369.90/warc/64807a969c749fbb215bce5bd57e0eb68b4f870f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747369.90/warc/9e1d5d3a471f3969f9208d2991788bacbc5783e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747369.90/warc/b0414a2f69389f46e58f5201d1efe7808e876202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747369.90/warc/f3f7c1d1170ebd9d500322b730eb7060517d6a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747665.82/warc/064b4586125b7d34ced8cb16fb787677b54479df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747665.82/warc/289c098d68b8fd3507592ec87566ee7caa823ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747665.82/warc/4f60552e0527f6c6c5673222c40be9c8d02383d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747665.82/warc/64ba2129ba14a7276b16738458bb12a88c481a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039747665.82/warc/a0ee9d2fbbce0043c79e4db424fa8ccbce13bffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748315.98/warc/2ac1ba8c27edef9de2ed6b6eeb571d5eebc9e105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748315.98/warc/99a6647c70a4092d9042310932cea9e353a9a012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748315.98/warc/bfa22a56a29ceff7c87989ab8f773d02a8b9defc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748315.98/warc/eed771d274ed18b5ab9555f09e9b0ad1021cb172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748315.98/warc/f182c4c85c6e38086170889764636bc4ebe33ea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748901.87/warc/0dfd10eb460bbccd40e1df45af96b1a909f9e419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748901.87/warc/1fb70dbbe8e5d6e63abca5445eb5fe24984e6c00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748901.87/warc/8609d68f7492f9817149a56b9274fbe825014d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748901.87/warc/d83d3bcc6381e5c7344f2540a1fade594e1f6dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039748901.87/warc/df4562f716a6c927f93dbd976f1f74a5a777dd1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749054.66/warc/5754ded070a23f40090b6cb3b774a1476ab99cd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749054.66/warc/6414f325c4371ef2954ce553fec8295a6939d4f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749054.66/warc/e8852f0f135b762aefdb424341a9d878c9877350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749054.66/warc/f852ed8ed7e38ff9074b3516e91523ca2c919829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749562.99/warc/075d4b1f0736864bb906447cfbe68066f408bafe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749562.99/warc/23f4997a7e7d6115a33f5c223d5d3ecdd853107c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749562.99/warc/99b6d6f78620c7744a37c8edbcaf26c2918db5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749562.99/warc/a4ebd8301430a7dd21b2b49f142d08f1ba451496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039749562.99/warc/fedf260c34b8d31c66488a88f564ca4165f0c828.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039750800.95/warc/4729f9af2278e5c6a5d6b13f03bf8e4af8aba41c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039750800.95/warc/9a55457855da3baaf79cf806bcd892f30341e515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039750800.95/warc/9bf9f348f8309c360168fb225e1e8424dfcbf0f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039750800.95/warc/ca9f2427ae5a9455ea6d68eac7746f1df2f452c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1542039750800.95/warc/e073e9e74601743e9dbc0a210e6eb063a96ee11d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1742b0b80b91b01c49348b9cecdbbc53d8735b12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1815e8bb03cce18d2bd31c6694444b7e98257444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/190a92b692f2811a2085cb8490dc4388044598f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1a6906102182cee3c833e28617b154004a4d5d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/1e01e23b2f58ccf4c18063e8b065afbff336818e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/2205166a35a0d7368cecbb391861b2df9999107c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/226ef60fa42769e8b1123479eca747e19e5e12ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/267a595448dd93d5d97938181c9fa9f86d73c1e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/294d78ad44a9bdacaf3a995429af3e84d89c6e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/299070ad45207016043e92755d32095e6a691f82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/29ee7fd360183b2fb72d53e317465d053cb8598e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/2bee311c02bafdec01cb4fcd2c9388cf58efce11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/2bf3ff3060ef3bf8f7da18d51831f948a1482a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/2cd029d9d9664fca98afe847883bd72db3702285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/2f9f1a108e5754d12544087ebbb8abf873a96bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/34ef348637d01738e946a8f95c546ef71c66dbc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/357f4148dda3787aad3a3ca770e8d6431b7dc0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/3fdcdcba3425f2b8327d6733dc09ef3a5dc2a998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/44b26c64c6b080bb97ed6ad934eb466b92f7926f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/538570ecf76f6d2a465bbb7511e7e14da0f2aae6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/53a1e1b7cc89f484852034e528843693284a7ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/54f6029b790b454fcda75f0693af83427f59ca9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/5e8f995ab435e95ecf196b4e3339cba19ae94dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/5feccc32f96d5963ca72641d16d7a7054021e05c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/601911ddc16648e947f61572e8d81e886d5f8620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/60f1f7e6a8a551fb0707036128d257d9a3985156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/6751f06b7f49f0c91feb2e163c53122761dac3a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/67c69a765abf7407f44254c68000c9db8c6ecea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/68ed6e1570bdfb12824a84d7eaee4282763f2875.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/6add3c0f60899314723843a181cf156b3bfe11e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/746c34ec20b3846ec8814b359dc52a9543b162c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/77b1777eccc8de701a73bd628762e357b412aa2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/7b655e22647099aa61e10c773c8bf6e9b8c4588e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/7c067129c034f1f388e1bfc0bcef48a76d762160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/81be23971b7f72b4282fe84379aaf35938c82c23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/89503d3521db30d8ddae530779e3f3cdad28ae7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/90b7f57baf182b9d8c1f6df0568a5bfbc1023d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/970d2070a6d0b8d5e08212f039a16c0c1399fc7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/98719e8c700efccb7e81444c2dad6382610208ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/9e12518b2ae191ab6c040b9df2f221a0a7f867bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/a520f4203b8aa60abf057a817cb95a600cc60bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/a99056fc233e81a0b841720364ed9ba8ba774b11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ac3be0ae35d4863d2f74c16574d556013e3c315e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ac9f817053ebea58cdef60ad597f13232dc7cd8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/b1a69f49f8953db217430c7dfceefe28807f4fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ba41ee819a8e65cb41d9d44d1303103bd6468ec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/bb1d2ceecd1e7dbe5e6b817c79d36e1b593ac651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/bb1fc9d2d2217f1b63ff86bebff5e2fa45b3abc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/bd8e22840a90fbc2c57c827951e9a9d79a66b44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/bf19bd3c0e0c9e6c9b5a948b58901358f2c54153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/c881c0fc3a6eb270bbacedb31efad3053a194e01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ce0b06b42c285887aa84b40abd67d18a723a87be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/d101711d38633b6afaefd787a6b5653925f66643.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/d23dd0f9061715163d1460d7dcac00417fe16e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/d395feb767e0af02a25950ae20c67ca31bd816d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/d49d4226bfef97f4cb19ce77a0a4c0f7eeb0af34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/d87c26c5479f9669e00479b8d7e9e5d90306a224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/da05102e63bfd76857610a1958f520febafe9dc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/da5f43564899c273d8a104685b25c4f3a174916a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/dae7ce9091cac2954e2adc1e128f860a61369dd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/db97e8a83a8c279e6f597b2ed6b1082fa1559436.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/de13a4eb7bc9c5a4c4df532e2c02e9fe7aaef930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e06760f14d9e42dc996bee4e99f346d403dee3c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e19d6fe8ca3121f0c180211879578d06ec918454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e3a654bc69ab788192f5e1bd85c24f54b10a1376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e48465fe93ed6a0174b6874666693800c77cd904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e4ed248274a2019e561547a713ff167f48c2342d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/e7d49c678c8aefa4a18dc2167393838a5af3b62a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ee065341c310b498550fe346bbda0501fb5ce9c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ee7b45071059707a7b32cfaf4dc03d740c9c5563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/f0bc5f4831792d6a38b31631ee8f95ff92097fa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/f0e37c9681951361a8e465e8d811be7def92b474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ff548723fe4a3004d19a35ddc6316494cb58498e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/ffa8446f2bbf2e2638b94323e8fc50d41da4698a.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-47 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-47/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-47/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2018-51.yaml b/configs/cccc/mixer/CC-MAIN-2018-51.yaml new file mode 100644 index 00000000..ee05743d --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2018-51.yaml @@ -0,0 +1,748 @@ + +streams: + - name: cccc-CC-MAIN-2018-51 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/00517976eb757f690b33f0707cfae4af288fe928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/0111a1289e97f1454997d399e0451e0ef3c886c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/061e06233b94c203c1ae74fae20dde15eab42b12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/082a51f8cb0f552f4564e8d8239d0a7ac602d7f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/08e07b70432419266de8e23f716f6f58c4b4a850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/0a6836d1c0b38d98a53138eb12c2bde22f8c25b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/0af57145d4f4651b8d040157a977118c278154c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/0d13aba5d586b9b77c52c9cf07d75ad9875e7fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/0e5cc3fb0a28558846acae2ca46143d3e8d04cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/11cb201601a761fb0a3953a0423ff8d681cf9027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823009.19/warc/5455e79fc12db4abd935a9e21a56a45dcd88322f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823009.19/warc/61b4c0e8f2080429a739effac29a0f9bf9d6008e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823009.19/warc/6e7b76693ce0f9b105a1c251e31d89436ec86053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823009.19/warc/a46249552df19e07ed284985e566b6cd2f7c867b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823009.19/warc/f00f4f65532c6a3392814c7a131ebfd087158174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823183.3/warc/46a8081216fdd481e5e089830b4e0b202ce396d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823183.3/warc/5b776b9f7747fb382090a9dff106da2af187a059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823183.3/warc/8bd988afae1f4ba68858b810b389dd354cb7ee4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823183.3/warc/eaafa11b4ffcf0046969bcd8bfc6a060447f590f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823183.3/warc/f2a2a52da5946ed99f4500426287228d7f3d3ccf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823228.36/warc/56b4a27012903ce2db69ea547a95de1473ebce4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823228.36/warc/81abf17657c71978ed0f5046d09853ab6ab92e7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823228.36/warc/9660d0638fe0a7a924e4cf0e3d8decfe18986710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823228.36/warc/a77d1f315ca12d523b3fa1081ea4cf81a58ffdd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823228.36/warc/d411107f5fca72fd4f5b13ff8a10fe6417a3b96b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/5e50cd9f9911ed185d5863161c272f5cf667d8c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/655a5def0714112f9399106764f5a6edb8280b00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/72cdfa60db9b9309a6e96f99c984f9008e35911c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/765a3096a92c9392a05679d316e0ce8aab652e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/a4f4748449fc850b40d00d596a42cbdfb797f65c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823236.2/warc/e9762180d5bed5313e296e1fd29e24a719b075c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/067231addfe5236f5ff3e8b1a6cba9870a3b461b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/6837a611efb704e04b90de6d420a5ab52ddbe23c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/736c48aa44703e0fde68d61f97746708ee070fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/dccc8b019b0f045dc44adb62b01b5286ad811301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/fa58458972467e22e6acf9700ed8a244d7357272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823303.28/warc/fc497bc9f30ca826eb177af82dfad1957be50fff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823318.33/warc/203ad4b7a880108fae59401a0ab8010535138e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823318.33/warc/2824f49f59966ae8d2cac16ef9d581c704b5098b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823318.33/warc/48a98e3e44dff3fb15e3097b6e9b314a5ecfe0a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823318.33/warc/58f50de1c736fbc1acb6529c61319576c56c2e21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823318.33/warc/cda3ac5c772389ad117bcb34806539f046dfcbc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/20d5b5d8c8a62a72925cf322183c0703b9072f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/27beff01db814559b92864044357bbe233178a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/9502e1da75a20582690d9bfb9cf017470dcc4980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/a76c6fc5249e6f21c1e84f97ab4437021666e883.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/ac62258a25152563623dc420df99e0ee4e720e24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823320.11/warc/be984922e421a9902068c646add247c771cbeec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823322.49/warc/06b0955ee4093a1baa135ab351e7335388c56cea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823322.49/warc/3461668deda0f712b4cb5a231eba018f28be84dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823322.49/warc/4db87483029baf9e7efa7615db828344d2ced002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823322.49/warc/7d053501833980352d9da56ae35ef3b4582e7fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823322.49/warc/ec27b2ee7b877dc8c1b7561a591915ffaa0b1304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/529c1e19078b22eb877ab282355f96fc9c51f1ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/68dda93b01c8b464c3b6692eccdb31702d650371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/995b63812e6390507b133e054f5306b51a798941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/a0b2bcb9568bf827007221018e94321207cb89b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/b47747cd5106f453d84c7dc9a78db938954b4d29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823339.35/warc/e431975b7abe6ca9da99f3b1971b0618f7f36e89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/37240650840fefed818d511a92048a5a4908774f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/47700158cbc80fa135adc200d34a1860e5c9405d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/5516042da1a52ff5e56146c45506901bdf9f8896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/585b6aabe00de6f18a31274ecace2b383f1d6bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/a571848a8494342a65567e8b40b4f0307df7dd5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823348.23/warc/f816785f1aa4054152b02f38d76e34d473e7e139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823382.1/warc/053c61a61d886e06c756d6eb616f32b30907fa3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823382.1/warc/6c1a86a928212360e411ba6c17313d42498b0e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823382.1/warc/cf7d2df6f247ec415870ca59eca0533566187f53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823382.1/warc/d12490954846cc200f45406a441e1191485040de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823382.1/warc/db2c3449546c4b8e3c029c7412cfd5061ccc16cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/36efaafcdaf9e4d0de3894ffbd2305ab9a68ada9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/6cd9f20daea10572e33073dd57039fd8e32ca4cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/a50cd08a0d86ddc8d44346e02ad57aa45e81417c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/b1fb9a568a19ab12e16c61033110551947788543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/c3f224eed075aaa4a911ad8cca73721dcc8ae6f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823442.17/warc/d574ea62dd6532647793b2863656e9b5a17bd6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823445.39/warc/0a4f3f0649cfb0de9c0cf8b7a7a5845df1d24b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823445.39/warc/1ef9fd511489257c7039bab70d5a5f6e1956fda7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823445.39/warc/385cf6c046840b0d2f24df8e7756acccf51ca191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823445.39/warc/79d136b74a7510591a55a03edd01f09fa188fa35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823445.39/warc/e5424e3fa956164f4b6f0202d9368062218ce00c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/4f7d3420194c3ac19f4fb7990e0b77f819fee7b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/62f4a7d4af309c6a17c86defaf13344397ad148f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/6d97a5e98a736e4b90514f6fc5e6dd1231039dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/924fe3ae5c64b2f22dcafe816fb90c5e3935a036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/b149386d7e5308e89ece0312cd03da8417c572e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823516.50/warc/ff4726d05048daa81c1afe4cc019d1bd0f4d3113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/0b6f45294f3976dfe008209828d6508dd27c04d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/13eacef3677744969a78327c8612a82d9fa6c0a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/7062d7964354e16a5ccdabb7c5bef9df2c3126cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/bd2b87a07c3e7e9da232cd564ffce6a80ca7bd02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/c4d1e213e75480ee831f9bdfe6116a5f4fb3ad68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823550.42/warc/e3c3396d62324e1c1eb51674b2cae72018c061bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823565.27/warc/05daa45182bb18b8e3eb8a0ea65c594a9f5c4484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823565.27/warc/4f665108a76b57a9cb1c246c44c79df07dc40b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823565.27/warc/52f8c97b33a6eacb22d18d913991493a8ad309bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823565.27/warc/76a9444ae88f778b68093c9888f2590fdadd4a08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823565.27/warc/e0b34e6d9555d043995b475666d088d5a2cbdcd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/67a3bbd1a38fb69389cb9b7d142f37c341f5fec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/75d04fa46817f869df61a7530ec4000a88107538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/d3f64c8ce1f4a48b3b0acfb6a6e8cc0283d2622c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/d5deea4a5997de14d119a05b034937b5ea97b8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/f29b4aa5e334a5c3cd07c2c3a57cb0607334b92a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823588.0/warc/fa6d4369a356c1b0c4014540d205eda083fe313f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823614.22/warc/0bc5f98832b6feef911f4cd251bfdce3b07329de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823614.22/warc/28ed3fd64e90a9d13cbe1d80057a189e7d18da37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823614.22/warc/8673190c4cdc33ba0b9140d15829232eb9d98c24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823614.22/warc/ca4452888fe5dd3e7eda7f4343f2488898126360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823614.22/warc/f9a5a1ace7b09f8b33d28f04113a288c4c1d0f17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/23ba24bf83aef58d22f13bd04807f2b6e7ab7a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/399d28c86dee9e5eee65b6d025fcb699c2c38dfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/6394faf8f8d8c15f484acbabda0f107ce25e1745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/b2f0a52e75cb298cea3cffc3ca197c040123f8b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/e0763748d49651e95be77368cd6b33689b9b520c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823618.14/warc/ff93bbdf095feb022bd3d92980a719f51dbbd04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/171e8df42e98e0ff80b0629bfa3a171445ee23a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/199c843da32d2a6b2ea44186e620785a798dd916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/43bee05d2638bab8e439f8216004ae695eb6f21c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/be569afe7a5de79088b0ae4b11101866814fedde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/ddca38627b472056eb7ac65d6b953a4a18298170.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823621.10/warc/fcec3c1f81e901484a1ee879b233fa6ec13a4580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823657.20/warc/0faff119e74b598cfaad83593c1871385202d417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823657.20/warc/50759bba2603b522aaaa44cf318fd7747489308e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823657.20/warc/6419709d1293609aa238655776d8949bb4555306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823657.20/warc/a94d486b50a1149cfd43d0c0c7890ab46b083e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823657.20/warc/d29d78917899976da8528d7942580b1ff44f6b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/0f1f53d22890d8f33532a332ab3982f3c2f512a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/2dc7b551673fda54594682ce68f9bbd99fec631e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/62a696378d54f7b97e81c3f0ba3ceb6c826fb9ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/64b34723ef042954a0705ed14375aa35659c47ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/a3aed517d88b6ddd4708d13e9f6f2acadc013fac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823674.34/warc/d543ddfddaf18337482b3bd531570ef7f61a8921.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823702.46/warc/481c7d10127abed8dbcea2a567d426ba5b14a87a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823702.46/warc/b2f4434095b2f880746c63edccafc4f780f65a9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823702.46/warc/bde95529837b34cb959db6018ad1ab36893e20cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823702.46/warc/c18fe1fb49079d190c65b47422bb99c90c695bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823702.46/warc/ee287b41b618b2ab0feb15c5706b2a0aad79eec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/72216f9460a61a7cdd0cdab69368a15ce2fd2a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/7c311276e62253bf3293db45e4d23fa5d3fbf99b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/84c248703922ac16069f8b37c56c31dfc74eefc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/acd8b3684901bed8acef613b1fdcf09c37a7add7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/f4817902e20a6710abe174e03fc9abe47e79a96c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823705.4/warc/f85b0f0f893c09b485e86e965e906a45b8c39b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/128b937bf4d502f0c366de7ce319dacbe1ef6c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/3d76bec144d3eef23830247b8a25f5204435ad8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/5b461047ecc20c8efa6f35da2975df0322e6dabd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/c10228a31e3d84384ecbd983d01f4e0d7171eb38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/dabbfe502556d9ec338e0c0cda87e3b33d382aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823710.44/warc/e08ba1dc33d547b77f69c9d7467e3f2b4d08266b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823712.21/warc/35910300ea02ef690d1f8cc57bfa2a218b61fa9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823712.21/warc/50c52cc7563d62606c32b0216b260bfc64a53562.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823712.21/warc/9113b4ab88e9f8510208011caf5540e65d8d806b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823712.21/warc/a341309636ffd5fd0828878d78569ff0ecee32d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823712.21/warc/f551e36309f6d215a15a8ce42ac7d22637558eac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/1a3f241c05b78761e2224e56a2931d9a1ac820a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/460bc5df7b867487e6d8317735db7081fe5a2019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/96abeefb2e119cde579c755006811dd8122941f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/cfee61bae5e9f7d0b2872cee239dd76f2bfac442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/dbe964c336eb55692264680e55077c0351aa83d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823738.9/warc/ecf703ec366aed0556b48a5796fec4cd67be09c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823785.24/warc/5c8383ea80acede7518dd2bd41daa6cec87971b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823785.24/warc/8b252d148321d3dd344dc8da2acb2925e2ca201e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823785.24/warc/b83bf49236ae27b99d5b031e580e62ffe87dde33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823785.24/warc/c931f04092493b7c979cabec2f1cf4a63b6151ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823785.24/warc/db47b5320592a16bb3fe79dcd7e5e649144b8bc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/1b3017a34dc672e860c4d951566fd1803bc8b7b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/304e22e856e2f5812812b73aae055b5ea44a192d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/85f5f0e45aee7d07522a757608b0b7bd68dd0ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/97385a1e126ea865084e170c28c1d405582ecf46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/b18ed94c0cb295ed4cbaac91b76903a06e8918eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823817.62/warc/d5e751b1ed2f0816096e28b1b7c8ffe911760ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/08658b212d242edcde1770a7b17a51bd72a26a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/0fc53fffec0dd650aeaa7b10c327a8bd44a79b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/19a8f779edf7472cbb9f79087e0bde3a69d4e3d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/76bdbe9243087e2da51917c8122bee7d2e8a2466.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/90b875a51f79e0aa05b09cbeecaa6f9f1677ae94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823872.13/warc/aeee7ef4b852370553ecae07550f53e70e26c95a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823895.25/warc/355eca41b6d3baeac00c8fa181cc0de1c0c2995e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823895.25/warc/53f28791a14b82aee3e7c72a98041195d682d689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823895.25/warc/83ec101d2518b3c2b84dce8c931b56364bae6bed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823895.25/warc/e683c920c5a5a13589ab6201aea6832bb244f3b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376823895.25/warc/ef1ff01b39cb3112fdcc37bf41e34443331ba1c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/16f555988763df32b734eacd867a6327befcdb29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/18ab6859a640d03192a4146ad4f15dcf2112240e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/3816476c13f6815d692caf9e0bbc54bc7b2b3491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/64419ee42839dfbc462b672150b7f6d14659a241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/7d4df69d06e4034dcb7b501c9e083cfa1d618fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824059.7/warc/afac030e11b5b504ca15e86d0c587c50290cbc24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824115.18/warc/217fdf7df20956d2594a60dd175a0b33321397ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824115.18/warc/5bbed90f532477dafa112b61c03894afd404180b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824115.18/warc/725b7ee938e6fd5ea7559c19b41481dc5018d554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824115.18/warc/84a516f389e3295c23e63dd92611cef300182a1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824115.18/warc/ae135fc4336afdb08b150a3e2a7a789cbc65fdb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/0cfd2269d687ee9559d41098b9997637f0dead57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/1d37ff4fa4b9eb5610b324c51b7d71d805fbd7ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/22f0c44572134b8e110c3e9f8ae65b32ea8ae35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/77b54dcb9f9b3a5932399be8fa1ec5a01af2ecfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/8ba52fc4a91116c55a2c52f0883d10cf43af9602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824119.26/warc/a9571c9d6cc800d11bf2b4ed66928e9a4f030431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/1caeb0d5e222be647a33d213e2e89cfeb6717207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/b03ce1fea26850787249ddf8c22929cc37c7633e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/c3536a8b7f773df701a891b26397f39d19a9f9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/cede9e3cb48d90c91dab4817464dbc738fda91b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/f1a09543dbe10c5e1f3aa8a6f41f7bdd7aa0a493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824180.12/warc/f2d8ed53ed40b4f5ea2d8b0cfd9dbc06841bb744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824338.6/warc/362fe3d9d83fcabf676ab7a231c76d7a45bedde4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824338.6/warc/8100f42fafbb230d57578102c95df753ade26941.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824338.6/warc/d9ff479a38b27b8099c4657bd245aa86ac2bf528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824338.6/warc/f04bc2f4b20ab751ab858042abfcebc66c91261c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824338.6/warc/f7f2a602c0b46cbcf721924323344adf1b08667b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/42857c5069dc093bdcef76206723963ca8684e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/4a9eb210afba87b12f0045113ae4cf3e1acad556.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/4d469b0c2ba55afeb88cf8cf87cf46c6396c4816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/584b824fa567f6bc13d2eeffae3cdfcbde51c8e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/83819e75452f398b08970e99f7888396265e810d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824448.53/warc/8cbecdf132ebf546e80232ef3834b58003071d57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824525.29/warc/05fd87ef4a40a62d5a03a9ac19fdd83bce6fb011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824525.29/warc/1ed7daa0c5452964d1d0443be29748fe6f1e788e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824525.29/warc/46f056e1cfb6ae99dd6d873aa295204a00366008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824525.29/warc/ac9eb0f941436be51107bd5f75d34e4546f23aed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824525.29/warc/da2aac21c45325145c6f90947947bd6aa26c7c12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/28f00b5999530099a694756c0072519496267d71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/2d9561c6e6b6228a731084f44046f755ac22d1ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/3da0aa95e75b5b7fa903c6968316d2f65b6d4724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/4e90d3e760a4921ed26a58535598f02db19e1b99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/6bde738777143ff2f2af93b579d56244b98768d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824601.32/warc/e272e856e2dbd2edbcc9b97f27a78133bd4bf5ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/1440ef1f840346ee3f0891b5d5b981c119601ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/260c0b8fac44558eacb71447f5057566fd8061bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/339dfc01bb55c6a416b82f7888e16435db8987e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/77bb68d2b7fb060c098bc24998f44954e896077c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/9a507090f2dcbde4107092c36c0130e8fa17bb9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824675.15/warc/9dfdfc50c7040ff632b2b5cc61b6d7dd34a51cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824822.41/warc/40f32883197eec830d93f9a9e3c029bdb05a7bf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824822.41/warc/641e68bbe59090e858f64e77f6d8ca7e42ff9b81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824822.41/warc/ad220c5b742bed9564a992de09fd0e19d2994274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824822.41/warc/cb6c4aa7660956b83c83139bab7e8babe8d5e534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824822.41/warc/d8f245de4ab46f187612ca0a0985a33631319af2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/075a791d8e2b524a1f18352599f53cca1be9acfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/5f142e6b03a45af8bf6177c2f32a05685c2bc785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/75dec9fbf6c37842ef9ca71fae1e2559b19a2d31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/8717cc104dc0c659e3106fb9d3985614f22620ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/a6fa73196628b07030593832e8641cde3b7dc096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376824912.16/warc/ce29050b7742701a18819fa952a48e889ae29970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825029.40/warc/7b6f22629adae7472c33887ff5cae0aa8b91d9e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825029.40/warc/828d1227dab3ecbe3aee6bdd5e235d618293340e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825029.40/warc/93b1bfa10256892169c60c6f9921c56ceb30b69d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825029.40/warc/9c3315ad2df00bcf97fb374dfbd6a38f1d90475a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825029.40/warc/e887d4d2e23eeb7fdadd43d2bb1d6ffafbc93b02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/07e8ec3c20e58c948bc925b26467175f5d2131e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/0a4f4c0ecf25d234a118be6a87e2d968799cc352.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/4ecebfcbde0b11566992633936584dbf8968593b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/b26c5bfd15929d431ce74354e9dee4cbe7309c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/bc553bf63d06170445ddd1b06420a7bafd867b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825098.68/warc/e4036c8e35a6e619719c0687051b1fa5732c0360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/16f4fb055028eb9a72176676632a90c69bd9c2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/22756ad21b9588f3885a9b5802e442e7af5b71bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/73498a4e3ae3a59da381c36e912c2743465d0aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/bfa189f058ee7eab76394bdb1f7c2820f7055974.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/ddfcaa3ba82d046042300e84b34f8945de23f914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825112.63/warc/ea06f54b56f0d99bd46fd94a5498332744dbda3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825123.5/warc/1cd4f8fad3770f960354e858c2f435f38415128b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825123.5/warc/5174b9308bcee3dcd58cda48c53282baf3d515fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825123.5/warc/79dd4f8c665d1e1593df668f218e7d44db7b3584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825123.5/warc/b946d03e5ac05ba4f1fa2d162e9c884f23454e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825123.5/warc/c235420468a314a6f8352fae0d3d57de8f68d90b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/31b87b0ade9cd7198a9c86a21211e35e52afc24d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/460326d90ec280f9e8d8c0637fe8a8a675d38960.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/46f7a8a5f115881ba3601e3bc6fda96775f9c710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/6587ff8b4fa79e5ed1f91381e9bc0a38d73e5062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/775fbdf8f46392d6ecea79ce0a6ed8fb42c898bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825349.51/warc/e8b2b34e0caa9915d8e2981559e6503629700d9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825363.58/warc/3be117d24bb9df40b297602962b8d051bf7c07c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825363.58/warc/8deb541812639b14d79c153162da41cc6deff484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825363.58/warc/91fb71513876afc172a51c9ab629c023e94b5ebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825363.58/warc/b8b2188029b46049604d2c1e0214558a7fcb0151.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825363.58/warc/eaf146291d3369af2413a07fd73f81553db885bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/0af7ea05acaad4cd4c41ff8112e548f2d4db19d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/72c7cd87e27ea4b8934d265993d59ae41b851411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/77172362315bf128d01d0701a0f038656f8ddd54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/8f3ca38f8674c03eb9ae36f98567349c04eb3b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/bf5ccb4491dc2a46fdf5591c74481bf2145f3a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825495.60/warc/cd02847636fe0bfd09480a4f97525f118f8cdacc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/1dcbf69b13acc81a13d2263cd31e97def68b52d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/4f1f252b6b50f60bb28874b9aafb6fa60b6729d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/91ea155f4b430830f1b7e72d5e20818fbc3af49a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/a352ae9a12d142b344c14df989f997b5ce59327f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/ecf971b9def56a996add5bc5b5ab5f39a6d0fc1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825512.37/warc/fae2c1411bda96354793329480b286fd786c2ce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825728.30/warc/14dd42ed5cc7b40796ce1a2eb7b55218fa224f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825728.30/warc/2e3e21d9fc32381c0bcaa37a0af72adc33e9b378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825728.30/warc/4a2fd196a17586a06521e1db603c2cbb1b68e49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825728.30/warc/60a7375b185693fd56423b35ddfbbc88f3b65740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825728.30/warc/8747d03f6fc83a06b70fd4306d7e12860d52ce2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/5403a2a12ad55c7249881fb7444b0a76121b1121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/6421668569bd697b7ed8b8e81d38f3a23ada01bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/9503378370f1be2376ae3d1dd6abeb0a9b3243d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/b4d1985d20b4017a5ee5081a4459f916b180d5e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/da25d7dc1007b6a367f2f29eab5f0df3e9a143d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376825916.52/warc/e766a9f07c760f684aa7ab1174e66a3ae360c0b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826145.69/warc/2c5ae4f434a561974bdbdf447af9c63e1d81b11e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826145.69/warc/6c4d0b00451942d63ecf5f3ac0a775ffb54cd45d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826145.69/warc/759f696a485c07d0ef2d1091e90e586d0049a44a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826145.69/warc/dafe99bd076081317e9ce5b1fda8bea25e08e15c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826145.69/warc/e8ae5ad5e33afb1ad29b8f04b4902026b0eeadd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/01af8b6d609576bb0e9b633c3459c0eaff0420af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/10766d134da44786d14849a9001c3267dbd71060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/7b807c0a2153e76518fdeb058eb0a1181307adbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/9d4ff8964e18bd02e09f3e0fc4d80854f5195ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/b62aa4b7f4d37187b151405af695ab5da429d8d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826306.47/warc/d52c26b3d09f6e5af3ed939942b98270d1e2ceb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/89bef972fc5f4cf60c28c4fdf3c31359d250801f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/a47cd31a692893db54255a4ea8ac4d3e67d4f8ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/af34b503c085dbe3d3bacdfcde263c5494bdfd39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/ceddb1a572ff5e5b2de31f195365e9f406d320a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/dbafbb159abfbd8422558845275ef2b84070fb58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826354.54/warc/dc47d5368bacb59779bf51ffe10f5976a644ab6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826530.72/warc/2b0b67cea458519b4d6b58d5605ddf472e39e97f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826530.72/warc/8c5c173b5c012dc3c218a4dd44c20b6ac8bcfbc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826530.72/warc/c20c73dc11e99372cffe8133e9d4b375f4fe529c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826530.72/warc/d045e0997dba1f2e8e9c562046853355306ae8ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826530.72/warc/f34eca3425e383f8fbfecf299cd1a67525df10a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/23d5cd9bf9b4fb13aeae9e9b6c1d012ddcf14937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/2e09a56d19027ae50a85b72982fb93c3c1c5b103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/45344f3b7cd0d2f74c5b13b6abbcd1cad876a5a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/466a0b66775bca4190724118dfc1653f3670383b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/73408ad06754d4a67d433079eaa67a78a9b65b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826686.8/warc/783dfddff23c4a3deaac70f6723fb8d0cd78ab9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826715.45/warc/39162d2f7b3db84f38ca17b56b129b574d1d0da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826715.45/warc/395e7a719ba287d677fe6bcbf3c98448fa88874e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826715.45/warc/3e9506fb8107fdc8acc45f610401484b1dad75f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826715.45/warc/767761976e2452a8e0080ce721b04a43a77bf410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826715.45/warc/89512a84e1993be86175d4bf50252362de6bc561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/29eb1070a5bda13eeab77dcfd3fee9ec6be1fef3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/80f21218ce1191b7111e0531c7d77b59366d0614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/eacf4eef0e7bb759e56e62250a48eacfdaa8f36c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/f36fd5a924ff6249ad86f631d56ff4e2749dc5ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/f7595cc21930453601cf208aa6e1c8eed1912c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826800.31/warc/fd4fd558fc935920af650fbcead4543f71be4fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/12c4f4995ded3d05a32a3dc253654f8cd7877b68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/14afac7094cb71e4ba8e76b5d1edb5a56d8db8d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/6c4672124475ec4020b1cff0d5487e77f68621c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/80f2f0665365a79341d96a9c6d6d3eee842b3b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/885b50c24d20a1efbc0de32a5f9e19c6d131ce5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826842.56/warc/b02359419426062f8698f2225878613221bb5bcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.55/warc/01b3356388ca65bae5f52424cef6aed7ff9bb3f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.55/warc/5e5515cc11b3742c6912e504d0ad7759c51956d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.55/warc/bc434530808511f8ce9d9847e5a3ee093981abd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.55/warc/e118b77df87d3275f7d7c6d694a3fb32ceb2e143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.55/warc/f46a43686b146cc62d7fe93ee47e58238bc3cd97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/01c1cf7613ca0c85c2c6f9cc53697e4a4af94d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/3f1c84421181a52288ed5fdb371d04dac0392792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/463d00b676aa236ceb29d18195391e373c2fa56e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/acf8f64e8df53b881e17d0108ab3be3e4841815a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/b16ec9925b3163280f37720f7d9bde0f7ba027a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826856.91/warc/cfa5f67c48e8e6e27aaec76ac9fab72740ce511d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826892.78/warc/09b2c9a6230b5124c3bd96d16a8a463d0ddce9b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826892.78/warc/146f5803ad15865e565989b049971e8d599e4ea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826892.78/warc/48f9583f673898cc2c253e5832fe92c5e427bab2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826892.78/warc/5ae535f9490afe34810d2771b156ce9bbfa5c739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826892.78/warc/96828b3f56ff79bd4e808f0ae3ed18aab0d89283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/1f1f9076ad166ee7afcff79186821afd8d820c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/49c3eed1a3497cbba41f92d38f5da2e51e224c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/91430f563845457c89f6684669c6ca2ec26da5cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/9b1a99163a1163e8b3bb54484d699782a31354d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/a7808f24b3dd21f1d83112c37026529e2e928ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376826968.71/warc/aa29df30cd10ae4be09feed4aa77c5a29dececbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/31baf82a0ae9b07dc30541a4097abe17f1942707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/3a3b4bd4919b139fa24088685f967301229cf09c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/4910a2894ae6701a29fed84feac62b7826524c31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/8dd8a1b497f2acfbe419f090e022c7ce537b9757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/d428ca9f1852c6800e613a0a9569c747bc0eb592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827097.43/warc/f6f9163e19ac287cfc74f3ea32c7e4bc11350e34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827137.61/warc/07f8a6396ddc4b007e206c9f44acb2f3a7361877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827137.61/warc/0f5c6d03e89fe0bc47e2ec358977f9dc465b508e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827137.61/warc/712e583821445fd373998ef0c90444c6d6a39c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827137.61/warc/7c9e45c0d8ede4d9f5aab1b182af276e4a4a5cc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827137.61/warc/98b2fc51f5fd0279819293d81ee231c5076c32d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/3d4da31087a0079bb90b2639bd6de84cd5a5ac39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/65e2155999680e110f44bb1efa81b41a56e6b7e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/7846026a89399bb71b4227bc1694e1e653d0145b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/cd31a005d727233533f1188c64780d782c992f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/d904d709db8223b01cb55a4ff2fd6152c20a18df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827175.38/warc/e24b52120e93b4117d586689ccdb78b4bd63e76a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827252.87/warc/9643a0ddbae4a2c1d47823f4bb1a3754f619d2b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827252.87/warc/9e454b4105d9cbc885328c643a8755d67147ac08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827252.87/warc/a43838110de5980b1886ac2573c91c1f0119e3cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827252.87/warc/c17428a6499da48e578af59fa3bc9c38cb0122ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827252.87/warc/e428369a0f00523eb6cc02924cfcf8ecd1c2d321.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/1afd1f75d6d60796acd3287d0791dd7f511169c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/47bfe15262c023cb511102de79c9323ba6fa71fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/6a5d16e72bec0417d36b6e1fa1fddd750185596e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/78e77885e04274ebed487d8d26325909bdeaf265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/c893d99bdbe1f60a6edbc39871f89fc0924679ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827281.64/warc/ddf854dbb7d30c2e0c3910c9c9330129e11ad3be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/0e8c8835dd447c9df097e67b4a7c83bd399f2253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/4f53c96f637077a5c7c3deed4b217fcbd8f7972f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/818b26340e991f327719e75b19a33b7bb4fccdd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/af4de54402b5813fd01a7ffd5366a42500ca8515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/bc2c4ed11b36098a28f0ecd599669f4a9937418d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827596.48/warc/e7a4296e25c9d89ceadaae94ac7660d43b6c80dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827639.67/warc/244725af6119b330da53979cb9968d119608b283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827639.67/warc/2da3a8c61b3523cff5859882b99e768ef77639e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827639.67/warc/4291b2045233e455f2c9a23f81346a4c72bf71f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827639.67/warc/575bdc02082d75ed2803f599ef154dd0a82daac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827639.67/warc/8a74e71288c2a5991d6f9887e00704d3eac831b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/9eeade3fc65650c914213fa6b61aff8d86f88907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/a0232db697094af95bb72b2fe644c882b792e8a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/a21e63ed78a2817d1959433fcc6449b4f23d5c3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/bb391c476828cdf5a5118f358afc8c410e7a1e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/bee9ad53c0b8cb0bdf68b75909a6919b5855b778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827727.65/warc/d31a0bfb8a186960c680dac362edf11c3db89ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827769.75/warc/728030ae5f05601db6425d30142a587cdc0d74f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827769.75/warc/8997c7bd65229ad7ee5283a16fd5b773893fcb9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827769.75/warc/ce23b3d5524559246dd23d83e1eba20b81b795a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827769.75/warc/e1a8ae2394e9e1d8d870ee35b577701e3e0a0aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827769.75/warc/ee910c8edaacc8c4171ac0dd5c5b16e3a9143e64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/6b4158a3b4eb81f745e00676ed54823e4a0778a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/c19cfa4d47df6da049d2e126f7e32a7d968857aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/ce19f3ad8757a50a1fc7b3d02e97fa16ec746ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/d1d11a34ba3bef6e23c82525d9b7fd9449562398.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/d2624ef4e1607558cf99bf6d681a17f940124e65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827963.70/warc/da631645df2a609f909b8d2ff47824973ab2faa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/3281a3886349d9961d494c9493ff7a973e61b0be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/4798e322e611586c9c622b41d455ef767b677f68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/51dc715e1dca65eb0f1a0fe7b747a06d1fe0d23c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/ca82da62a364e0d2871d26dd76f7b6349c2431db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/ce5f75ec746938cc433230023f837396c4f9f847.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827992.73/warc/d8cc90b73c6c00e0af1aed691ed482e1c3ce3a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827998.66/warc/4047cf814d7546c443ff225c64d9fe3c4999b058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827998.66/warc/54361080e7d33428138d72a6b05a01905b4cd607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827998.66/warc/8c34da8e8b0dc5490f1bed23ed64f546d9602c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827998.66/warc/a00c3e23d4bfc5ae2cd51e31beba38f43bcd8a5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376827998.66/warc/ba1943503d20b135b38d556f56bf5458a554d8e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/206de0f4bd829bddd26a00c1b6ce74142c6a1a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/35f0a1dd22a1d9e2d201bd444dae7cb06f39ae35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/4a40240dddde2ce20d84e2d719e931a9ef6c299e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/538b13f1a7ef7d4953f7d008e1e3b92e1333130d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/874a707f85f5d55e571a8f7e8fe0de2363ecc52c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828018.77/warc/bcc482986cdcb59d58b6151313d1342b6368de12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828056.99/warc/2faf5abb09f5c69a83bacf00214bbda0d436e837.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828056.99/warc/4e2fba66e1d4aaa3c6a313e3f45ff5ecec04046b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828056.99/warc/83f6fcac735df41488e2f5770c7efc31d70b6fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828056.99/warc/c3f80c655bfb425f2e2b734ce4c6ec00753e0445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828056.99/warc/cb4bcd89127a20907d443e29d00ef9f2a2bd3a2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/12765419589638dd809545c3da6a6ffed8848c1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/1c01bc3eb693f15dc8cc4270606d80be53296c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/27edc2531cdc6b0ac9f62ce28fdeeb29e57a09f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/97eae660f59a535c17ea75d8cee4b92add2b89ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/9b7fbf456fe6906708474f09e504ff635232c64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828318.79/warc/be98577c5bada97119fa848858ebeb545d43c937.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/03024d501fc301f9b261fc41cca1d7667044a107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/44600ff9e66e62944e5c36a3832bfad919a06e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/49706c4057317494321f18fd2a37d11ba5c16cec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/4b33f936566b679efca81ac3de75120b6127cb45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/60d53dbb424792ab842d647dd89a25646749a764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828448.76/warc/bc0918dbd7508351e41e3e7cd357114a4cf89e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828501.85/warc/6f0b86b0761dfe6b5b0d9726107f30889ef85daf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828501.85/warc/75b391a40b4192546e9ee6f2e8fe6f1e7de578c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828501.85/warc/b183bda4fc31c98973e2591f9f10c94cc706ef8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828501.85/warc/d897e4b3306b3a31333a0ff6e8d3777301fe7704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828501.85/warc/e4a7c624cbe6c782f37e0d27a9eb28a853bf8ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/3305319d414a567cbe98086753ada6d00122a57b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/4c4ddd9b7434a337e80e09c6361d20c6bd64b55d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/5e527419a8bd88418f3b4407bf5babf2e5861c5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/6d50eda9403a6c374cb6c441e44fed092a48cab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/70f9b11b0f15b72b2856207c5b324b948a1eb379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.57/warc/fd42dc9bb2ef3e704046975029d84e207dd9a0ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.84/warc/18c4bf86808ac93093fbeefb84117858b6048927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.84/warc/9dd0dc4c88424549442f704d1160057c1584afb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.84/warc/b47a57934ef8b56da9995d7f936aa4483d696c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.84/warc/eb79b20f170e79f4acbe3f4a8120ef989790c657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828507.84/warc/f461da6267c729821034becf90ea27ad8808de68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/07651299e576227fbef6d2dcbbe0f3e7776bf4c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/0ccc21b6304a707e52c228ce3779dc9a981a3077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/3b207be08e02c69f399f580a9d5c7a5df31d4591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/cafc986a158bcd3f7142035024c6e4ee29dd4233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/eb676fecf16eb73857f2aae1c6daba587a59d816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376828697.80/warc/f6e2424783f47896fb27fc5ccc1d5aff3f7302fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/1ae9e41d1c7f194def40ac0b46cd1e960f325761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/20511905db4099361f7cea13fca751271b235b9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/320e12deadf28b5ec03b47b5099765feae75f737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/7ba1813d40657420bca3fb76294bec30ceaf00e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/d04d64db307d13edd6cc25021787494ce5188b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829115.83/warc/fd115b3ab3864b9c4a70eb5ca2f415041003020a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829140.81/warc/0061e015c4a4688c19e8abf35133a3e35acc90b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829140.81/warc/45a217a2b702a3436dba4727f2fc825a1eda3ac8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829140.81/warc/4ca398fd4024f906177a6efb2987020d781ac302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829140.81/warc/7387edefe571885f32c54d2a190e54283d7e6125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829140.81/warc/db8fb25af44de9c6165ec8142610cf0eac996c62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/2a32484a54a138841906b00f6194281cc4341486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/4df2bb00006e6f10d2850a83b0c5d9a78606494d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/91bc5d5f487bb84cd877d17d92b3c1bdba42c67d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/c35d10303c44fe603fe6dfaa5801b8be49cb41bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/e509857bf4cb2c00247810a6971c8036ed2cd439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829399.59/warc/eba93e1783e143578c1cb74fb4bb4377d7266327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829429.94/warc/0b0e78c59ae2d00c41661b315fcda10142603c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829429.94/warc/2d557554ff96e294bb955c055fda6e3949ed0273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829429.94/warc/3c547e41d52b1182eb4c0faf3e99efc93e78a3d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829429.94/warc/5b59660783612452470588fce5c179089b76210a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829429.94/warc/9550e0739545586c19b5754b23a5aa1dd17908bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/0996b61a16f67f928b57c7b7c12c6e2d6ed97228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/2c35c6e3e5342a3bd727fe52a0e31444761e6e6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/348ca7409122beaabdc93845c3970e51a5c2c32e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/496b01f3634c1a89d5fbfbbc53dccbf049cfe895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/605e40c1f6e5e96fb598d351bd155843dd6be407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829542.89/warc/d2dd86fdfcc39eb999b050b4050889501f354730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/3f498d2c33c7982085635ae895d0852691299021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/535a24da989a34eb4479cf85b3eb20bd2cc944d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/6f142a66322ad2f0c53422960b8cc31cc58d1318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/bd2e0fdcb121f5b6da7495a92d43a475370f18b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/c21422ffefd15df8a51fd941ebdf8727f1502f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829568.86/warc/e77bacca6e82bea8a68b072f1398377888e479f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829812.88/warc/0c6954cb0b2fbf3d328d1c4ac08e4cfe8c6c29c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829812.88/warc/320d05fc05d9a8fb074002889d9465b6bd94152f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829812.88/warc/bf54f76b008f7e2f76f4136a0c980aed7c79f954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829812.88/warc/c7653781fd2e66ec2a2c9512d0f14b39ffd27313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829812.88/warc/cfdd4aa13103ebb50e97f856a290493e7cb3da5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/0fd4f55c2f3f384dda23d92bdabe29c4ed7f0a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/1dd27f120a3448322f8aede71bfcaa8c7398f407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/9aef3e7401a2508933e2bd2131002d5947e0a41b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/a0684c0ef3d5c72d83e6c099bb75bcf6edbf0427.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/b2ebb6de39ecefb8f6908008d75405cf1f62c61a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376829997.74/warc/b941e7818c63484a0dc3e0986dea12f3dfab8a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830305.92/warc/072498971afde526685193f285eff687438b0ec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830305.92/warc/1dfb45ebd9ca49ec069d337d61d23d28cebb165b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830305.92/warc/203e9310f75e9269a30fab5f4ee5e07e78a3b4d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830305.92/warc/5e4c68f7593274e6aa738156df4d12f8ded62057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830305.92/warc/88c37f217507715d34a721ccd7c09f5e119d2e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/1049d748bc512a58cddb32ab447fbd0fa3b18208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/411e08b315228fd8fbcfedc6e3daf966253dffd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/43c8fb520338c4f436481a0fe4c30b847793e426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/90842f90453409f460ede0b31c98a7ac8325c824.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/9d2d2dc31afe131e4ef1ddd907dc3181a30d0283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376830479.82/warc/e41ad7bb30946e1318377fdfee52fea0f5e4c289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/1bd7e351dddc932de88766cdeae4d720bc295d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/45d795943c5adea0e753e65ae0196663cf56aa4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/860a2df1330ab3f2b6e2485a4f8c832bdcb09269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/a6a916daeb3c893ca53ecf668cbe68ea69846635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/be6ba92c52abd59e00388e0bd89608e3b5a4d3c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831334.97/warc/dc0eca8afb6b0904ac6a47334e385f69dff7990f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831715.98/warc/13175cca8f1d021a47fa2d868c3914b34f4eb412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831715.98/warc/76b86ad7cbffe3f8facd2a95b75512f410e98284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831715.98/warc/c79794a2bc582999b8a9a04e3d90688495493b1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831715.98/warc/db0860bc6fe3cd22411c31399a47c021f3f4734a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831715.98/warc/e805669c5818db62c92d69d3d05635d7ac4ca862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/21d2cf2b731cf1ee7a7a13bc91dc46ec940b2269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/2a31cbcb6c6e63ccc2804c090582544833f07a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/636bcb3d77b64704f1a1906e322869d239a4de67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/8ae107db273d8d84736dedca5550da9b05a3a66a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/8ec44911d1bd5df4ee7a267b2df777e5946768e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376831933.96/warc/973479c6b6582792ed5517b148b6d6476385e160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832259.90/warc/0a96c43e31446ab557f55a83cc0647b21dd6546b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832259.90/warc/1e7f1eee040756288d5c1a4ce3812e0c794664f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832259.90/warc/295f8086eb729e7844125f3fff3ee68e1ac24f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832259.90/warc/3817509c5c915a57b11c84d5ddbe8cfd3c291148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832259.90/warc/4a45fc05c18d18eca46d8942cb0b8afd7d8bc49b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/166ae0e960966737763f0b360f80afd1f9968a84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/35f3a0859bd4ffd0a43c871524c6af95279b7555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/3e7a7b3c3e0f9517bea344ee33add4a8a4113a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/45ea03b599dcb23990f44621abb00c8c410f5730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/76640e57f6385c0529875b9ad6ee7ac6520f2ed2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832330.93/warc/a533e969b9114f7f35e707d02c302e77d5fa245a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/112539fce61177d94fc7a225c68d6f401267d7af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/2bb56057a1751f81d7dcd26fc0dedc5a19859044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/414e63d24dacb9e8ac6473e5b8cbe34978c47a10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/6c78753978f540d61f23c129e54aefd26e97f338.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/75a9857d627e033792e064ff278170aaf1003cff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/8159c9a764dfd50679d9f275573ec3993c4f140c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1544376832559.95/warc/debec76df9a43cec2be4d24aab0a3af4abc5ae3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/17b418561a9382b55d8f8755c70840e9e4ff4e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/1f74d5de5d3e5b6e47c43b3431c9d1de324e0ebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/23e784b57e42afdc242d3bda8e9e73b34a705524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/25e2ba0f6daa57ee45a37f24d98b8d9ae061ee75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/2611b18821a83588d036b39a3b3c0fe852ea8ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/2667c15a2887c2fd047f27f9d8abfa65f956cf49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/26f4b469e49a3dbafcc0b3eb15d2cf5cb76826fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/2a6d0acae041f29ccb9a25fb20e09dadb57dfc11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/2cdce88ff11381221fcfc943e582baf22aa90058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/2da86b99884b370990a21dc5d765a02d501e3e3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/3343edf5f76cbc56c50cb10687f7bccf2aea3d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/364abbd164047fd2886ab5b3ba86694cd09ff38a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/3f55d3a34aa98fe7e698d679a88548c81887316a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/426d70b774247276e31d0cac0adeeee35991570f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/43f00ecdf788aa943fc79e0c8db36932487b571c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/462ed455e4fc128bcfe75ec3eeb3f9686834d0b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/46beeda96746ecc5473278cf923852294dfa465a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/46ecc6e0da08ced64c3129bda48207729ab9c422.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/4d687cf2b614008abc973daa3d75f58ffacb189b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5292dd4447a32dd836c28989d0991c24222fe67b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5453d1113a01feda0ee28512e672bc420c4fbec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5a262d4a8f3d8ab8b2f0427ee1b9e828e5ac6019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5a70f8bc64df68935b36e2f37716839051fb369e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5b0249f53ad56d64a86359cf7a6e9e37d4e07354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5c74be0b1a0b8ef2dcc5a9ff94773dff43a02210.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/5cbc4ebf82bdade6c0fcc7941f22bd792c2c5353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/68708b190a6acf783a96ba8c8cd1d2c70219981e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/6b5e9bd32cb645fbdf09c012d7c159f785b5709d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/6fc8c30059dddbc3f430cb5901653c01e704fb72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/737c9cd69fae15113c37d347dbc168dfa9eb12a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/75ccc40a4c271b1ccffb8796c308278b33fae255.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/7741d848ce70b89d9dd23ffa480ce442cdd71ef3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/79c6d11e712adfc5e20acd0f52d8795c4674ccb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/7fa841ab3ae6dd2fbfd0a92bd7fefc91ad571aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/823df1d5afb4a5ca5e027e2861e3580709d3759a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/868ff774999d0b0c7fea5a265c86cdb4959c6380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/91fadfa3fb094e99e5f87d77e5f9d9ba318b4b2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/94daaa1783f2567763785485675a130808659dce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/95b416276ac8494583ff1cca06dfe84ab082f9dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/9bac4e6fe5613ad05770dcf84eefd3b7466e6209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a2ec4d23ede21c1d6c48947cc33889baef7dd465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a33a16ce17f2b21d4dc4f26d23343e71dbacf8bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a3db9902f1cd92e5e7000a4d961dd9dce79b875e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a4944d47e396862afd9ac474a32c0b156771e4de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a4adf1120ec3c9185d11b457694264f4acb21b4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/a4b9a2ea11671d8ed690fa39450d9b1a4cfd9f50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/aab855962727ff94d67d894bb18854ab403713fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/b18d1745ed42988d284a9054ed92efd172f59162.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/b4b35dab1c7bdc020278be729e1e5213a2b5b62b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/b82caaddd835ec2022f16c3d88abc5efac505361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c02ed8df7c1f3adbbad1af34f9607990826358d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c0372a00ccb98668e46dd14c605d7e53735dabcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c04befc5853e0bd1f05ef4222dbbbd1dcfcbfc96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c06b1f250f25b5f105a052a71ef6f4c40abcd706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c11677cb29c3eabffa758c8042870583c960a5fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c1318a6047421ac8e09d368244565c10c9d4ecc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c8ede41cd2927a3d943cd0e0283aa589a5a27135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/c9bb73033349072356a2a806c9c02523b06bab2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/caa01ea8060c74c77ef6c4ca8b3665dfbcc68857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/cbc9450a652af6a8a8e7e7dc4b69606af0f96b02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/ccda8ed1570919447b4fd6707b33f579eaf55775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/d09733f3cf90e47864b98b9e6342fa4e42fc8ea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/d1c0d3cac8d3ac41230d6f500e949cb2dca0643a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/d6a68b15a98ad4a4c196365ce87c3a8481ee8dd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/dfd73290583ee25927393c177d0fdf02c7057b66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/e987583c76c79ade747fd971443cf4e4cad92a67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/f3fa06636a480703fb01d29832f7d978909ee3db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/fd828f9bfa608c500787897da4fb75768cfbbc79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/fe4df91f4ede80abcce44b19bc213f7257dc7c96.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2018-51 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-51/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2018-51/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2019-04.yaml b/configs/cccc/mixer/CC-MAIN-2019-04.yaml new file mode 100644 index 00000000..1c4bf64b --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2019-04.yaml @@ -0,0 +1,754 @@ + +streams: + - name: cccc-CC-MAIN-2019-04 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/01d1fd8ea49130deabc4d3ba0248f0c40cb7137d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/06442b418d89ba606baf453e96cdd0c0a8420ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/084ac20d7250427483f3ff6deae5be63597a648f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/119d3cacd3d2633fe9254fb5f5fbe89c5c6c0836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/121a8edf7bd6754dfd85d0b38a2b242e0fe9eb37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/49368059255e877db2f59f2a8a4ab9bcd26ca256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/726cbff98549e0a3c0b7cefa3a58a97d010544b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/76773d9b1e3bce965d187e6aea9f48cd2eaebf74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/b9301b4fce7253cdfad9db6faa816321e6afe8fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/d76de6c3393dfd4675b736ebb588855a9f7c0817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656530.8/warc/f896137b71b4571ed3a928f79429cae6ddde4f43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656577.40/warc/19191e61b30767420f88e7392f861ba02ed64e7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656577.40/warc/572233b6ecddbb68aea63978d1f52a54561d7030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656577.40/warc/b43698935cedb531c068ea12ac457dda6018b97a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656577.40/warc/b4a6acbe9198e75fbf45ac40793e159021252903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656577.40/warc/f9748bf98d8ea4ea83312c9064b6639b5f006934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/213e80975c7f7574dfc7279dca3ceb4fcc4af591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/2819d31c544d761fb9250acc8e5395e42fd19702.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/2c3078d139f53714008bf31ca7d54ab055a75337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/54939dad447e362a87828b1f1c8ce2d20a6ebed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/7cc58132b01d919acd653b9bd06dcf84a44b3725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656665.34/warc/e6fc3097abaaaad6888555e70ab3c915a72bcbf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656897.10/warc/6a3b977231b1d7820bce9ec4ad4804a515be2506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656897.10/warc/7861ecd3b319c647d11e42223d884a7c974f3cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656897.10/warc/8dcdc98722d768c88d99fcf0f0de360976321a64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656897.10/warc/8de6a9b763d090faf5b3a60df1d5c144f248a490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583656897.10/warc/d42340b587a1d5cdf192fda81310fd1e91f9703f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/5d9ed56af027f2ecf93fa3410c237cce6e0640a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/9cc41ef0eb8a1a9dde5409ae982ed927217eb1f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/ba03547b0b72e4ddea06b577c93fc4122318fa51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/cd24c62e69aa11dd8658e7c66c9ff57d3bffdba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/e810a96d592b3cc1f4c656efb2c6ea1db6915d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657097.39/warc/e96c10da1261b8bca0d28d2ac377d33957a7ee1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/0305fe848292bd786efa650a9b08eeb4aff6d970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/0a6ed93587aa31644e6428b5ef1f4d902fba4eec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/0adcf81cc589362af6e89c213419c540296d70fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/2f8e47e61d63ea5c5596c08f44fd3dea5f2d8aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/9d19dff5b8f34e7927e8723f9c72f89dcf6e8eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657151.48/warc/e40cffc6b258be63537bcd4d282920030bab6e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657470.23/warc/04edd6282b9ea0e535cf211a8b3833592d03caf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657470.23/warc/60bb9392b391852cd70ddebc78a81e34f25caf23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657470.23/warc/b758aff560d5a5c5db7416be3dcb9bd811becefa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657470.23/warc/be7dc12ffa6a0a09c40e7a515d8d238aaf6133a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657470.23/warc/cdcb30c463296cf97b8dc8e56c8a774d6ccc9fb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/06cc4215407c3e03ce14981697b4a91f881a155f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/1bcc87c3a8083f11a1cab8c45ec71bda69c4ce5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/7a217d7e65cdf1558c4a22a1affca5ea39f47be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/b9f037aa69dee98ef1c31554dfd92d0323170155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/c3337392659b57fa5060eb339adadccf2f91579c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657510.42/warc/d366547db674bdc6803472d60e10449bd97e63d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657555.87/warc/54276a381cae1a8fe8445f9f55ad51553f8c1c31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657555.87/warc/55efce6243f244fc0e50d79f7b341f49e5591f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657555.87/warc/b8d2dd2d2f78717e07145072b6f1a8d9345b9c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657555.87/warc/c9c2117fd33e49f367021e23b939f0ad389d1ae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657555.87/warc/e05c56f8b678d7404ef04cf788e2913b2a0d64bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/03d839bfe190177cc04d23e14ca66b82a435f37e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/2c1429347e88a534f9a4486b3dd95894563f5fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/75d1dbfd142d04fc9921bc821196d32e5aeeb52a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/a67ab0200b934e6041a267c4e1fe6f850056b3be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/ceca9134caaa2117a3e6e919964627eef2980dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657557.2/warc/fc2637dac7c532ce33492962af9f1e5d600960fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/3340e83100292a5ab44dcda8638e34c45953af50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/3a9838d5e25d0cdbd93eaf5d04fcf8b51c2af559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/698a299b7b2b0287400321833cd4667a82ce4f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/87c79be27fa7a16e6514234bfb1cfd9f0dfda256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/d9f4408d4c3fe383bc0c4e1dc6fcbd9df868ed19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657867.24/warc/dba068d51e16499d47dc6a2cdd48667e16b60572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657907.79/warc/58aadf3f3ac84e037e9e91d694b8ffb87a37be29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657907.79/warc/6b381f743d2904bad344b09ee9c0ca43c68e46c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657907.79/warc/86463a910cadd114b93a996918b14c6338159880.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657907.79/warc/ad9e968f60858c685d5e7ca4f429344bb7778284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583657907.79/warc/cfebe0e18b1856d1708f85945299fa1cf614d725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/27a05adf1bc8e9e98a9a8f8c489fbc702dd36ddb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/2ae2a8ecd3b828ad3b7ff35c0c2245ae29e7e651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/c66cf903af86f03a4e92c06dbafbe7474904bee6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/d1f8f94d88e9c7337835e35090251a645d3828ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/e7a1f872ba1b3190f86416c33a4f0bb28855559d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658662.31/warc/fe025a41dfba5a3c2ec6a9e68da13d07bf7fdec4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658681.7/warc/0dcfb8b1f6f5427a0f7c95fde31aeb9f3db874ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658681.7/warc/19a54f78f03c2f8c3d9b62e2b24a93b7318b013d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658681.7/warc/485589e77255cbaf559daf02c1d52bb010606cc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658681.7/warc/c0d264d85381d5b1c2677fb8ef7cc5449070ac8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658681.7/warc/c8d856d7893022b605347cfe33a551a919c0b09d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/1cbcffaa441d2c1f1df5b595e1d746a8d8b04960.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/39ef1e141d9b12910d9afe780134635dee99e5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/4152777a1bcddbb71db27b46db5a7e75bdb05f46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/476400db3bfccbb767160922285b5ddace58106b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/96b836cca3fef291a3d5129ac3886a669972c622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658702.9/warc/a63b989518af0b143e4ac69dc888e786e0b060a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/2971842eefff966ec7a2d034f9373b089bf3e64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/39ff2e9021633dbb5c0233b8c1527bd38c3ee211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/419ea9218fe8579fb89688bc3e15cf1cfcb1210c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/be4b6278eb4fca9c2c9a6a4b0bbec76ad5c21add.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/beade28e99ccc3348523a4cb8c340f14777f6f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658844.27/warc/d927e1b1f464f4027ad8f8e49735e57915e5ca4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658901.41/warc/0003e6b18fcbf7173b18567285bd3c2c67f5365c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658901.41/warc/1e85feb1c766a8f721e6fe9b338ae881092a08c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658901.41/warc/2e0d607a5f051576128c0a00d8dace4d63cc54dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658901.41/warc/70500e2c3bd1e3fe9bc2412152e75934825763ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658901.41/warc/b717b9b034a8b1c8803dcfc8f7e361c15db994bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/0bf8ba0613134dc68191e154a0f7f27700309f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/d397bd8342980840626e87d632b43d4463bd1ead.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/de62629df2605fe666acf110af27f0c6ec230909.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/e513cf3b0841a713ba2761231818a60a12756b0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/e7e70a1553a315901984546af19d473526f7bc88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658928.22/warc/f389c2e64dc36aa27bdd8586c24307c5caeec5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658981.19/warc/099dd2de27ae4819465b0b10b945af94aa4e241f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658981.19/warc/9fd5c70d3b9519f21ece118bd48994ac83bde74c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658981.19/warc/caf294812f00464887ec9618066f549397eff5df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658981.19/warc/cf692c6c9009e28b4bdf165157f389618219f5ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658981.19/warc/d042e0aeb975ed164fc14ffb8e591fd584e4c88f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/12043f0cb1ce180b7a50f8ff85625a31be0f2140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/86536a683669a301dfbad373a952252e5458465e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/87fa253deb94347d902861ebc3bdfdefb130e716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/9de3e7a54ee973b87c9af414619c7a8ac3d4383d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/d0f4112207ee61889e9aac408b6180ed464cea42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583658988.30/warc/dd2e7b30172386d84da97a48a12d4776df886bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/1b665af3f981a0b6122d967da48bc7007658e4c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/79fe4128c904959dbca8c29a03535798862bc218.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/91accbfdaea0d1d98ee1b1f6d9311fccc016cd05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/97d2297d817615926e3e74f4bbb6a68dd8022f2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/d88526417147882f26877513ab8fe3b57ff9dc58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659056.44/warc/f27d2de125a53ae1266d4da6a88a81502e4aeec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659063.33/warc/0d5a4b606824dbd80d45e9ede658c71218653e4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659063.33/warc/1e0aa32b74505438a6e0190217435f25d8776ee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659063.33/warc/604da6985d64f2e153e9a7b1db064e8219b9e73c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659063.33/warc/67200ca692845e6063130e2f4b8206eac00ed0c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659063.33/warc/cda9b3549bfabe5e72a58fed58a695f27bd2100b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/0265806333ff30fb2c59c85bf6cacafdc532e635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/68fdd2766f2567837dd613b5910d163bac66b717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/8b84f8b009bc6eda0ba4465caaeec6709edc7e5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/afafc1e123a5d661dcbe8fef331d1814db978808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/b5e821d9b751f0ec4a69332f2e826cef3918f076.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659340.1/warc/f0eca7e7f1431807e8789035af3ca2dfffc9fc27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659417.14/warc/2dc57fd417adf6d0a40455411966ce62a53627a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659417.14/warc/81df551a33ab768b5fdfabf418c56a3f02b5dea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659417.14/warc/954456a91ca033b6f5f3db0771187f10d8f3736c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659417.14/warc/dd0cb0d1151995ed89395e32d4ae1300d084dc2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659417.14/warc/efb614fec35560cf5640b21f7a2a7e72b233a06e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/22eae9990fb6b33737e37c13a07a2d191478b712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/6107bf89e1b2b4600a770fb01e48e59c6d9a3e38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/80e8e015575ebb49507bfa03190a137121712a14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/84922e99a54b69a6c2f37b026f10c1005b2e073b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/9f169af0b9925b267cbe658f5dfc17a82dc9bf0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659654.11/warc/e5d5f1aea462e5c2cbe9fa83cfbb043ed821bd40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/3725f00483e52f090138ef9521942900d739109b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/c270401e70de87301336016667638758abb26d81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/c74fa65bcd10322ac9e9834909c4ce04d03e466e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/ed0734823751ca2838263a720f23729a36052db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/f8d681718baa85cb7ff2e6d082b0552cfd443ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659677.17/warc/fa972903ca5437a5d532c49b7e404b40f097c790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659890.6/warc/0db74a781c0bcee9925a3f6b49da6069c3d43f7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659890.6/warc/3090a7d1c2920dc450ade31dc53e0fa842b7f878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659890.6/warc/60ddc98fa92f483dc917d2fc3d87d34e13c85582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659890.6/warc/c238e60ba161fd4821c073e89d42eba0a89f45bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659890.6/warc/d24591981734c23bf14c8cd681664b10478ef305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/46b6bd92388a8df524d9fbbd3d6080706752e42d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/81de295504df4b0365ae0f6c5e71ac7c29fc0a28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/8e5e3061c628550807954d11028b97dfec16f0a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/9956ebabc1bc96baca3e9e5a569a81af15b4f112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/ae1fa200d16013c139997e06a8ce6e2808e1a73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583659944.3/warc/f535af07eeb62c65f347df4060fe6afc480fe968.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660020.5/warc/1d32bcfcc37a18928a23000d1292964a637d2075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660020.5/warc/362d130be56b785aa7f316c5ef6fd18ca0b3d280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660020.5/warc/5eec1b6cceef694db7cf614f71c1f20f2bf07a53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660020.5/warc/c44127439b5abac4b4dab809ca6eab4348ac2855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660020.5/warc/f8bdb505e7a6e3f3900802e6c339662ec7799ceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/1391668454baf8bde2be1d701b728baad7679b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/963960bee3bc2a2e2a1933925edf6f9e207d0dbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/b87e63ca925cefe7ceaf06ad1cddb8a2a41b7800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/ba5dd2389ccd2b6de81efeaae5f14936f5da4ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/c90981ad5e7606f8a10d550870e9bcf551d1bb01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660070.15/warc/e44691c3c07c6fec9cffb17a83e0f06a441d0a4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/286e5d0bc6db6d888b77c8a3f3d52ce02d28d22e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/2e2c776b97caa99ce757796d2de0895244e0be18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/89c3e0c1bbdd28c0dbb9cd4ae66ffa7a6d8f7b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/924797b6e9b7dce2154a506016fcd50c59f0eff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/a564ef002f5e78311808bc6d4ab2ac6fba9e3961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660139.37/warc/e1d57a92ade4868c08190dbfdd6f43e0431055c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660175.18/warc/598f27e06159fee3f530cdab6d962a78d5a80b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660175.18/warc/b9f8ce10a8e48adb060b37a18fd235b25cab49ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660175.18/warc/bfb6ed68b3730a7260f07b678a7bd1120cb023ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660175.18/warc/d6f48c92b8aac04c5f67f443713e7df035e691e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660175.18/warc/e197324f65b546d8a34562cba03490ce94259e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/0f43f7659205dadba3a20da781a9a0ba66c14fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/235c4e88eb6dd843daeb34c3d33bbcb0a3320846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/356de7b155c747b81d4ff896f0c1ba6704c7d2c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/61e0921f16067c787e506966b23492815b600f1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/bb3657f5cbdc2cd63be1f5639190f356ccaafe6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660258.36/warc/fd731024694431523ce61351cdf75f2784a115e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660529.12/warc/2517697a35defcd20cc7d91e576ebbf9aef0ba54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660529.12/warc/30b6f2d2927bae359310ff51e2338d0be2dea30f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660529.12/warc/5e31c740535d5efd606835b1f17539a2666ebb48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660529.12/warc/9df0a3c9234525cd6e707dbc49286b83b1148bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660529.12/warc/aad5284a1a8ccf884146cb7ee135e46df218f836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/6246c1c08226e3c13ce68aa1dec8f99d9d440b33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/687e2c9e3e963b107293e2d8afc2d7420d33d8d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/b08d5c900b2174f0d7442e1d61acc8dc3d148f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/b44cd88f9f5fd7ad0ab2086a843532a2265deff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/b8b2936abb10a4271789a3ac0dabf4386106d5da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660818.25/warc/c106b6af0f7d846fc410f1ce69b9800014cf5021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/09ad99fe1cfd53dd21c63383e98bc0211a5d3bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/26e56aa895b239a8c8264b20198513a971e2c68e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/434607e9d8e509556ecbe2f94f6606347bb5cea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/ceb42241c949b2b0b71f176e130d9fd9410198ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/d3fa89a6b1037ba753d43969224b1c3bfd60c763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583660877.4/warc/e99d9eae810818bceb1f91362e3e5d57836f658c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583661083.46/warc/05f579c119b2c7c020c6e1380c99b1bf7646a535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583661083.46/warc/4b4db4e3f77a788a96a14534452f9785a1bf30cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583661083.46/warc/903d2833c2d497687ddeebb50aaf32f6d247b52d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583661083.46/warc/c07b71faee97a59deddf580902d4e9f59c8437dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583661083.46/warc/c256e909a00402d4a5456c66dceee43ddbd12eac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/0d0e1890e8820b84897dd3d7e0702ae50308942e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/1fddf4d5640addd42e3e63889502e8b9d1bb2ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/270078a4fd81ebef185874e6107e03ef1174f974.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/726abe745a7f57731f790738c5806c2226aa8075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/826c752249a38835e5b4b3f75937c89c90d98f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662124.0/warc/85412a475b941fb600df0e781a524aa61e80b3ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662690.13/warc/026658916739a90d87f59adae949386ad3119ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662690.13/warc/47100de8df394f0d411fa583c927d468f6897d8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662690.13/warc/489ad14fdb0506bcf3998d720503de3906aabf75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662690.13/warc/85dca64f96767b153de9551bff267ec7821a04b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662690.13/warc/fea82c29237d8a6fc8808b5914c934e29d314352.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/00922220ce19c7742b06e8737c289275faa3a5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/1fd8bab0fda223a2c83341cd6ceb1d16aed99c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/af454d4c4ca644c1a5edde818d869225f9304602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/b1073d93ae56c176f998d2368eb5e04b22169efe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/cec9092b5379a6cf39cf8ecd0af25852efd55dd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662863.53/warc/d645c65495146b392a24253b62019798a8c9a1d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/78a4838980463158a5b0cb1f7c5857f8906bf4b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/7a4541296eb8ba7bd8005e97bce813b1a1ecce25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/e1bff4ad2c2072bdc5f5c50586efb06af78cf730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/f62f32db58fa4cfadd2c1aeedce822659da1ee38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/fab039cb53fbcfd4691e87561f6dda1eff386c5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583662893.38/warc/fca73432583b171f30256ffcfb635214affc66e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583667907.49/warc/60609a30652826b6431530f4aea776e6e5429cd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583667907.49/warc/6e6d8a7983d15f0d4da0fc0142f4a0c9951ed2a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583667907.49/warc/743eca286ea7c92c4ba2eff1742f678ef53df5e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583667907.49/warc/8fefc0caa50c20c9e6d96ecf2a277e98e32ab79e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583667907.49/warc/b3d90cd7235b146e61eaf7423bbbfaa625bbca99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/2985d084274d0dd605785d29805148530965d0a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/3c9fd48367ffae71fd879e1ef36d66ed398f7221.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/41f612a1d10e2cab3afe929a439bd74604a3bdf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/6367c79577f48bca005152b80e26a01c32231907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/db43bc4bdd602d37456b2c58fb6e3d68e25a30f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583668324.55/warc/dde5a860f723a1e4a5889ab465ec521fe8a5e285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583671342.16/warc/5a247058303274f7ad9e29840c05edf9e24c4fa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583671342.16/warc/7c522401539526523a5c4e7138baed72ce662486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583671342.16/warc/83ec9a04ada69e6cae22868f15f5e4e3653bcf05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583671342.16/warc/b210d875daabb7286a610ca80bec83c35d625871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583671342.16/warc/ce2fbd78207e7f8b98f886c45d457aa26378ff5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/09a5f12542e5880dbbcd9e4c0d6275a560e3b9e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/27b1eceff9800f36e757b6efca294139a7866afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/37c8a877e64ff39ca26f212e6265b62278fb5c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/49847de3cfcd64dc1469842abfa51fe36f069179.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/52934d3067fcb0a9c33159b9b3a669a24098a113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583680452.20/warc/ce2f1fe516972e8f2249b4ad9d4413ede6ab30ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/3bc70e198f77de92b6411a37b74d5e48c8d78c3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/3cbb69adf6a5f05ab05ddbd409a1118ba6d23e3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/60626d81dba474ce31be7613523ea4966c1abdea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/673527ea7c4a90b26dfd18fde82ed91f50e8bc83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/cb8f473be5a3e00210a750ced593c9141f6e1ed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583681597.51/warc/dc4b9a8f4c4884b09324935b400c57d5bad53589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583684033.26/warc/4a96c60d9cd9728c059669615d76d777591d7900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583684033.26/warc/5753ae742d1ca6938f5af177645ce4b254c0fa8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583684033.26/warc/c081cc845c13471399c8985e3e7925cfaa0d3e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583684033.26/warc/c322befcfd3d212b1282606d188d3fc091d41cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583684033.26/warc/fda664b5aa9f121f55dbc15d1d5ea5fc60f1b83b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/386b7348a3b4163e440327852897a866fc5c2759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/5bba8b1d24112f7f5bfe7d7842b5207ceaae5ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/a7247e59f91e90bcf7ba256ae477b9b09fd2d58e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/a7c21ac056239145ed404768cc8e64c72d8dc77b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/ca41050cfb449a04727ef957c7ccefb898fcdbdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583688396.58/warc/cac65a023f49abbd9d88a2fca50ed9141cd4d968.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583690495.59/warc/09a5a430f91dac21d6c269814b597de9be189678.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583690495.59/warc/6ccb48e35ccf564df35c902aa7c6a819918459de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583690495.59/warc/9fb46371e63e9e01e17b4aa3c50593629d4e0b46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583690495.59/warc/a8a196ccddefd0d3cefefd665976ce0c7c1b0da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583690495.59/warc/fb85d26b2b089b7f17353fd7e4cf4233ea260b14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/0d2de644eb1793a10c7aec9e07b5ffacea72e0b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/3d34e674c9724a7cca805edf20b462b18d588c71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/5f7d538e8127232f6e1c1dc9b04035a9bf28f85d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/dab9f8b9d36e4e37dcfe2d7ac964480ce6df8662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/dc2add068da9154b9579a62bf173515237635753.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700012.70/warc/f2c10463215081f7375034d3755ae0f386310545.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/5dbb6e2a99d47e75b1bcee84354cc00e74217aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/66b4bf796d610006616d6cda8a49ce2557bf59d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/70cf8886c19b7d3ccb503c2cc1332f4b5b47131f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/bacefdcd122573f72752a69e8932513be2ffd879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/db4f162a040d3c4c2051b17b9ab5e76945e53b51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583700734.43/warc/df93e3b8e86b1d6577299cc6a9deb1314560d78b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705091.62/warc/a6e940ee18ec1f0c3f00a9984f3d426ec44d1a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705091.62/warc/ce82880a98c4f57582492fadbb653f88dd58ed1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705091.62/warc/d6695289c196589b760e3bf0d6907036bc297182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705091.62/warc/dab2263cd40144a50bf3c227f940dd325dd05b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705091.62/warc/e3a38adec56850ddb6f7a15de7d5c7f6f01ce551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/16b8fd6413ba7c98754e14d9c1413c04ce399213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/23217d8c2d49f9978d50d0f4e48011ef0bd63e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/2df13083db1b9088d96599c918a97a1ee6c51cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/56186aec3b96f1c73352ba02798f9898aa3770b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/6c7d22d644e73d936517b192f19e13c8921b96fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583705737.21/warc/fe9c52bb496b1033512d063cf6cef02c80b96ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583716358.66/warc/23f2d3d57b971d26f288b274b1506784183830d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583716358.66/warc/3bf0103b823b9c80922992edda5136b93d99a948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583716358.66/warc/9083c96d848c640279f2de07aed42c630d440e73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583716358.66/warc/c48557c66d1c987a147133fc68a0f78bee486a7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583716358.66/warc/ec8c04a23cc7c407deabddae19dd9ab94120fa4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/12c828b6175f9546f2303ec4110e0802199b00b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/266de9dcd0d2064970527c136a6c1829bc2b2d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/411e8ea920cb457d5bea72b160b5becb01fca565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/4abc7e05ea4be81fdf2af0f7f93fdd29d94942da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/821b0bf9cb0cd8eaa9ab3831bc00c91f3ff64aac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583722261.60/warc/98a534a30830524ea9beed6c5c2e490bca4edf53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/290b50af0f1be73e13d1b978e666d66f37717e29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/794f27c5d86ee384d8c1cca5d5f44617ac44fc94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/9fceb8a237566707e656138e61672d286ecfcf5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/e6a18891915338c9ace33c87e073a631a382212d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/eb0c8f3b0868d65e944954b3588a022c8a51155e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583728901.52/warc/f379969cb2928489a811747e9215174a96b5b1ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583730728.68/warc/7a964e7e069e8c02bbde2dd3fc0a2e425a1cb317.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583730728.68/warc/9c8c04908b7de07c374bed570efc935a1f71fffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583730728.68/warc/f872a68880b916dae7ed6732ed7b01410b217f9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583730728.68/warc/fc324ebe99e714a2abe01f4c7fcada6374974228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583730728.68/warc/fed32931b7ca4b9c31cabb8b466c9889f859097f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/301c7bae802cf1c54bfc3e00d208594c4f8c7d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/6b25e3663beda7ee6076a39df3478814b89a24b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/8a1ab2faa31ff62260bdce7cf1f194841d2b03f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/b1535771149daf60fbb0bcf5cc3fd6044e1d3a9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/c5288959e64344e13cde10ec321cddd452167c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583739170.35/warc/ec938b5a974777917358b91abe1effeeb7152d40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583743003.91/warc/51ed3eb3046e869fdc4c875affa2eaa88d817771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583743003.91/warc/78a002b7472f3158a4f35231fd326053d5647e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583743003.91/warc/9bd1e657ece86a763dc86ee794caedc909087d02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583743003.91/warc/ab9a81056e20b1c26a73034fce8d8ce6ae8f4db4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583743003.91/warc/e36170806918b56e39aa2da1dfb5423a8b3889c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/15df9c5f01b13a1ef0e8ad267515c37814250a39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/1718d0dc2ce8423edd2703020b650ad91f1ed8f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/1a4ae9539794064bc6e5eaf4673d9bbe7d69a29b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/6a596c54bfc32b44ceb5014f63ab2777063668ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/bcc3bd7dc3f573e57bcf3b77aa56d9674d6ef9fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583745010.63/warc/c580e5ed4937238ac26a76338465e09e4daefdc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/1064961f3de200933e976520cc830ba5692f522f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/2298b1af1dee278ccca64d3380070447e1efdc45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/c9001118276b8208e6b44addcdb62e39f6580a30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/cda6f0ed541fac0f701ca9af80d2ba5f4d618a68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/dd644ac8954af99f0e59fe5ffecc54d8a32a16f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583755653.69/warc/f7d5a8dcb2187c814769c22e29c2947f6910cbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763149.45/warc/29cf92ca5659587e92b7009f0d03f0bda9dd9389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763149.45/warc/3deec63329316fe21f14eb25b275d28ff7623cce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763149.45/warc/473a959857146e39a1a0467f051600df74634a3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763149.45/warc/7da4dfb02dbeec9ed392e74273f6cac10bd16a79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763149.45/warc/a8f7b119c6e751d7a985837c6ddaf0a68d204f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/35176be1ef05ec78ad5ebd7598dace28150ab1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/5730ef254614160a1a26c9a43720a9c3b2a9f50a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/728715102b512833f648593e07eb1844f3dc03dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/8ed3c63fd7325f929df6041824b54640566298d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/ba7831ae56f00dac12960ea3285e27dd115b4a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583763839.28/warc/f162381a1a8d5e286608c6c6e3b13df879b42e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583771929.47/warc/058d759860813f2616cbdd62404aa4a705992da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583771929.47/warc/5179a72f57927299e2160953d77a2feff17b45ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583771929.47/warc/76858f79e421c8b2bce9568ab01b05407228679d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583771929.47/warc/814ca3956f950484c263ca0c526f8898bb553859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583771929.47/warc/eb787393a006ac86d55cec1a6423ba91e34fccd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/2ffeaa68c2d385953b02e9ae6077a8ad3ef1af34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/3934a0a68db0535d1ab00c123bba3b863dae2a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/3ba35a5b922e283e5cbef4518661500adf7321ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/585f4be6517527c679f258022a04b8a869dfd3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/cbb899c3bde8364939497a4c9ffb36aaa71b7ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792338.50/warc/cfc1feb05eab5a3cd67d8652e48eec14ce4a0e8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/00613c1eb7c6098b354aea5b08bd7837bf700334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/5b34d218f0725d8fa981409263897ebc3361e373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/7569366e5a4e5a09a743f53c2ffeacbdd9c6cd8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/75eeb840fe51e1489355d54770b99a0f29fb34a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/9604d2e2677cd6049b81d62ee338b5bbe25835c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583792784.64/warc/b1b54d6021611748b849300381d9006ba20961e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583795042.29/warc/03f6d4c7f85fd7d37db6a2316ad601809b60017e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583795042.29/warc/0c6b87d9b69aa174d9bef2515a470161edfeea1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583795042.29/warc/a4b25e78561c8fd7b3f15865081d1bdeab09f2f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583795042.29/warc/d2808511f4412349cf86c2e35217d9eb345a01b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583795042.29/warc/d7d9eccf8383d89f4aae39950e3cb88b407673b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/02900c83df46ac3db1db85be568492636ca01484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/03ab5436c61982da5857cae5fa4f669904a4fbb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/7935ff4b3a15999e296d1f3d753b19933347b55d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/9e889f9299f3f87b1e4b5eff8d50332326949729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/c53a9eb39d8e46ef7e8c249da1f70e514f52c8a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583804001.73/warc/e3d4d84cfff4f56d92eca821e68779c327ac2483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583807724.75/warc/501ec110c44acb2e86d61796f7b2ac7f71e61190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583807724.75/warc/564646f8538438cdcd27dacf55bf08806fe73166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583807724.75/warc/5e8a7762452d7c230f6d0196c0ff1234483617d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583807724.75/warc/c9ef251135d4d86d755cdf3a59b96cf4b575ce67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583807724.75/warc/f12b86de573e0865b29d6c7460d362c2bc1989b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/26f4c6c0a1ebac2d737dc53b37b867f6572049d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/658a1954b69f27c5829856d9f2d3c6a742d60380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/7efb49f4d0d8cb73b8a8d40c4c0ed3d3b5105843.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/9407d1d4d9a7999885f43ad76779c56652c6bac3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/deb6f195596f5311e3fe43da7ceb2f877222463a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583814455.32/warc/f88a54cafeba18fe1b2282fdc8a3efbb50f6261b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/13aefde120f98af8fd4326b23a5ad4a1b43e0807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/4615f45158fd803cc4275e1d19cf2000f25ae9cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/4ced88db31e1f05d485a3b0aa7e69cc21f99f6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/656d3be51ecf3dd238491505ad1018f18246d81d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/97723eec5edc74917626a17ffb92d27e57e309d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583822341.72/warc/b1f1a008bff23795a55570ed1833dfb0ba5c1267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583823140.78/warc/01d2260c22137f457a98ce5ac9c3126b854bf96e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583823140.78/warc/7b6964fd0bd3c56979bc62e13c7e9a7304d3ed4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583823140.78/warc/84a79c85cb34fcf976131e27294d3f4351b1b80d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583823140.78/warc/b8a4bcafdbdc44d1b74cbac4abf77e211908075d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583823140.78/warc/eedc288d9b0d3627eb0924daa3ed0c45a49d28a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/11eedf45852b489a86fe4eb856e78f703ad303ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/3a89531f38e9cfaa7dc6ec1cfb8a0e0a4ce87b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/46521fba4128ea75bf93b4fd9e23428e0df183eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/71262c9e3b3d71ba750117f98fc7d164859739c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/a374ceba6a5792b7dbb0986faa3a0c18f7b47deb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583826240.93/warc/fd1c5e4d75899b28cd8d719b7b8192639b279bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583829665.84/warc/21b88b77517d58e4008ba34a4d3ad4a56b907b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583829665.84/warc/49625eb5ddbd1dce71777d024e7ead73720352ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583829665.84/warc/87a22242df2481c04b0f22c2cf9f5d14df1c1d35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583829665.84/warc/dd94f0073a78167e33f9d015121413bf37810554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583829665.84/warc/f34dd2b0bb6fa51a097680eb4e92dd280c204572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/080068fe37e9465ec6e0331d8b8e2b160c91f167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/2a624d97e59f472fa089b377d2c6bbaa1838fd7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/8e48fb9039cded1f42bd548033160b0dfe9161d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/acd61a1c371c8b098985298d5e95a65bd7d28557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/b785d4a589815136cf38ab0c3b42f0ca7e161596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583831770.96/warc/f052cfeed41b800539372540b17bb603fa039c26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/16731763bef6ab5948088dd2a435a28ca2fd502b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/23cf033382717e5239456f12b15f58c174c765fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/378706ecdcdb23751ab49fb6e7671a20f7fec3c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/92e789ac8e9302d36031427097add1219ae7d37f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/9a2962d12415c253d2cc183fa45711e11c42283f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583835626.56/warc/ef03f307208366890fde806b38d524e2e87743ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583850393.61/warc/0afa30e1e3af04e077d87c0d8f33471734cddb49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583850393.61/warc/0d79b35b34f802ba28f0184efee69327661370aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583850393.61/warc/2c0b8874e52febd8f1e20f730f8710e0f59d60c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583850393.61/warc/6e73178942d8c66e4a426a2aa909a92d8fee8fc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583850393.61/warc/ebb381b795d238804f73cecbfc438ce4033f7209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/77cdd4c46d3078c1ba686e5a4772ea35fc07eb08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/8aab5f2527c2246d0f68356a38eaf30cc7c20c93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/a7b2de59aa27c412a8d9839fba7ef33ec40ad228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/beca6a710fe31481b588f47ddc3eed8f1731daed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/c1c930e115bbca417c4d5289e97d63a9682cc23b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857913.57/warc/fef01ddace56c91009382934f6040702325a3ed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857993.67/warc/3ccfbf5fa116503075703cda0528faf9c49b4f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857993.67/warc/a755eef80aa2bbd83601ec021b3ac433e96f691e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857993.67/warc/b494e204f75f1c17e2de980a72d5a2fd974a0c5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857993.67/warc/dc632ee752bb6765be425ac63bc507aca59039c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583857993.67/warc/e7d4dc58982905620162e00e41153c8938cdc12e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/0b4633bf2b1c0607777814d09bbfee700a69828f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/1e77fdfb290f389de82323d0ffd40d245630a26f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/32d9d66e3c2db24e2d425011bebae6b6a964bd70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/49c24720feec0d7745a475069afcc7208b5a6761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/e33f285948f6e8ea5686c7ca0421da62b1a25595.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583867214.54/warc/e96cd3810ce12b6d894acaef0d6b25ecf8849ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/4658c0e98bc31c967350c72dd363de9119f64541.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/6aeb3b94ca2f5f04c0a0e7ad0585d2a4d1493796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/7dca629a98d2c30dda0b557316d24d33361dda8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/be6fa1dd1fa9868b5be765617402370e4cdd8ee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/c50245f1e16686b63d1304793e2120517e22b1d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583874494.65/warc/d840d70c53f8028b1c8a2ebb0a37af5728d6606b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583875448.71/warc/3d9f788bb5b391026618ba414d6e9bddff873159.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583875448.71/warc/7c7a4235dc893d33aec18a19c6def6210f83c04c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583875448.71/warc/90f5a64dd884f747d7acab7149773c96bbc79c3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583875448.71/warc/93d43f63012b4df9bc643afb3ca1c28f5188c0c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583875448.71/warc/c66d9f122b23f3923a3942ec4f8f888de1c4a7a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/6eaf726d6525c5d81f430c7fcb7daed05ead449e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/8baf25fbc7ff6c7063f07ca2420d1bec5ca71f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/bd48eade4013144d1134c27530aab88f54220c9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/c2da6c03b6828aa4b367cf47ce38931c41969708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/ef44b4a0a45ee3483619a64a7ea920bdb35fa920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583879117.74/warc/f15981c139899798cfc93dddbc5b4c7cda3851c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583884996.76/warc/55a18cf23b0aeb834aeef9ce07813dec3699f92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583884996.76/warc/70fee007f457df707ea0e534266e5b8ba6205c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583884996.76/warc/93097d9f8c1707491503d07681f3fe3dd6298b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583884996.76/warc/acc3e2b460e3f65525addf9086433095e8d5a5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583884996.76/warc/d3b2b78ff24cecf4997c9e3f8c3d480d3881561d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/5464d008b03eb47f0678510dc4bc8c53aee03f7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/5bdbd1495908c8bd3ebf56a8f35d354223c1ac92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/72b342a1aa4ee7a2663404c4342baa6d031fadbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/cf94dd291b2b6980ff7a235a7738c51bbd4c867a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/e2e079749d0944b5104f0a104bc7b91817fa53d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547583897417.81/warc/fc082bd7b8b0455d2f58bea8039827a67d2ab90b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/3737c5a3d0bb59c3443873cbb7eb8f6a60679112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/8e82a2e83070841e0913fe1ea0463eb9835ef089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/d17ba2302c5f9e66622b2243365bdcdc4bd2c19d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/d535a6c13c597cb35cac37d240c5bc8dc982478d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/d655d48f36b282e35d9122d8889fe676044c3bb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584203540.82/warc/ec6c78a37773e512faad5643d9d0caabf516a23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584328678.85/warc/08e644e32eec673bc910c9f61919587900121ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584328678.85/warc/77f4f00a5ab2af178033ae4854731151760a9e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584328678.85/warc/c150a93c5f0d5ce97790a7ae42c9d27eef46fa3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584328678.85/warc/cfa6ff8fa70ffbf57bfc4f2ef09b084c2fac46c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584328678.85/warc/f299d9f8d220195661f3fd1f1083cd185841cecd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/081f4cff98f368e01bb032c580dcd151b5749af7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/1d6393e6a41edd7944c30a90b6577a200a2308c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/3c149346e4d9ac3a76391af112f112d4a02bb489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/b7babbda7551a61da68e247a673076ef2d3250f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/c803a8942f86fdf81a6d72c3518d63398f0fe94b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584331733.89/warc/fd378b5661ea0f6aff43057e13310200a7610da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584332824.92/warc/1121428cdd9ed9f3bbd549ca2880e0b33152f38d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584332824.92/warc/181c831f60ace7b0d44407782bf8d30d02fb726b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584332824.92/warc/6c235f4d4d5536cb7f5779e3942f3bf16322c46a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584332824.92/warc/aa669ab2fde2909ef83aa924874b1a7d1d540e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584332824.92/warc/c448cd23530fc6a3760a18cf99f18a0913e29c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/212a419f4574aaeef442c4a0afa0aeb040ebe0d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/2443d7a5f6f65ff1556663b76cb7c466c3493018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/362fef2bbe3817fb137b3eabdbbf6b561a706f42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/a133c45dfc9f6002989b9cab34ed1389b614c99d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/f4413aaf79a0de527734f51eb24e14f34b89b661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584334618.80/warc/fa963f324376227134caddab41792991e2b22622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/3a6cba1874a12732b470770387c73e7cd6adc5cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/79af20b3102858f407d4225f8a1e322a4215e925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/b7fc8d296e3cab680043883a47161459d0b65c81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/dca10fd6ad3d290359f8da9f788b144993e09784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/efdd1c1c871f2e793adfb1dd012ba6c06399b7c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584336901.97/warc/f75ad152e9686cdb2b9176621042a0ab86715e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584350539.86/warc/17dab616dcb6cbae848db2ce17336d284c9e3a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584350539.86/warc/3c6ee8264af52f35906356841682f1775c07a690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584350539.86/warc/59715b076700ec2977fdbd547a5e4b59be23b3ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584350539.86/warc/a0af7da2907658fd673834eab57dd3218c4b6f2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584350539.86/warc/e8a3475f38c55723902bdae72c67998ff6abfb9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/18d30ab0bd825b8789f1227914f8c345413a5f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/4538cd4cf0fd2bf25a6e288c25cf472258dfc4d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/5fc8f5b4f3d77877ef8e0616dc7167d4035896e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/8d13ca6542cce0141160ba134af8ad49ae5cc398.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/e79a43da33c9fe7e7952242b130fb619fb8baf3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584415432.83/warc/f441cf1614433c84f287d0d5cec5dc7d2aeb2fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584431529.98/warc/0ce861ed699fc350a1c7c49d4228bbcb673d831d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584431529.98/warc/0ef9c446c2e069357bf3a2c219d79a4b12280ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584431529.98/warc/415bd87fae7333e526626e4eec688973766418bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584431529.98/warc/9bab10ef271c22df47801ed3922b7fe14913bace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584431529.98/warc/f7f1fec242badabbc28b3cb19835216c7b7d6968.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/4455ade61d8df0b663642dfcd9949532cb96232b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/56ec6f764d1e12f6d2520c2d450d580183d0cb8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/7c25667f028d6134a3cefb0d62a86f86ee1ae92c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/9c5e37e46a4adc9f22c80d89920d3defe71e669a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/f1862ca30bdce4b5eb1923feedccbee0b071f874.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584445118.99/warc/fce769feff5c847a0c2837bd2dba60c0bd94bbdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/400afdf6b198279b8d3c850531d8cf7bb310e0bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/4932bab3122f6c6b01effeaedf1b33bff9822dde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/57ed13ad5e8032662737c09b95126383f9a8d450.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/9912016b8556bdd10ad539b0bb0062bc092d4b5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/9cac2a6a9b971017c4a3f1aba16f672815d2b2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584518983.95/warc/c6949ceb4234d40e4f948d2fe8e7149a83ab623a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519382.88/warc/15605f21847848475e1f669ddd9ef486f8c2aea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519382.88/warc/4c0df3313efea5a78f5e911283728a96aae8bbf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519382.88/warc/90b52bbe4950bb30fc13f8665304c137e518e84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519382.88/warc/9cbed45a6ccf5b7816631b20118f5bfcae81048c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519382.88/warc/f80b9cf6e5eae273efbe137f5999aa92a72cd8ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/1f107a9e50dbb6a92c5933983b45c948b651e141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/65a783f7434d667d14cc2afcb9206c5f3af7b64d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/89a7cb8c0e5d4d5f83c584607b01541561907af1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/a1ca7726a4cef5636baa3cbaa1addb084e5e087f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/bad3ca3ebaa2441ef150da7e37e971276de78002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584519757.94/warc/dd85df4e24d49e6678bd77468e28ce0b72505b06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584520525.90/warc/4765b58443bcc5254f3480192e2599591a5c026a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584520525.90/warc/4ac95419cb86e2e93d018cbc1a101d717fae3c13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584520525.90/warc/81efd7410deccd185e105ab40e275e2ae8252ced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584520525.90/warc/c9120580ef48c7e8b880a53eae3e9cb59a86a7e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584520525.90/warc/e17fe148352c5a6bedbee1c49eb2bbba02efabaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/486f059625b9ce9a63441934745f4b0d16ec5e9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/5d02a3d1e2e9bb1f700e5d4e89a983684a6e9bd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/6df84fc8b936013610d39ae62e08511b50664e60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/befcab60e3c3ca36735c189df0862ce2cef0e11f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/db19053ba3b128571bff122ec1654d53ce8c3906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1547584547882.77/warc/e72d509088fc0f46bc2ee332b98eb90c51456ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1706aebb26d10dfff9af978371477105b97556af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/17e157459f90ccf2adc78457a02bc6989251a3f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/185e14694357faf78b1eb4be3dc78980543a23fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/1e14aabc1710d1eb33e97ddb23ad2b4ee35bd88f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/20f59fcfd49cb9ae905a9f7e58714295093615e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/247899b5a8f94eb49355d1bdebdf1559bfb794cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/29c3ee733dcee92792390724b7323a43a18aa8b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/2a2f646ae88f59ccfd82fd48d412a926b03a98ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/31c36d0675a73aadd7986d220d232c2912d6ee34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/367481c53bb0db2dd1ecccf72bf127570c99e734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/368720458bf4d1778d973e3c4098834fccebfde1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/38522159c09d55fb17bb764cf5f0529bed6a4a1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/392017923bc94f0ecbd7dcd29010373cf567b5e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/3e1f911ec3baced06bb5dc4ea55d9cc07f205290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/3f49c3368201720a782c40dac084cacd13b490fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/401ac628559adbfaa41cf7d20670d90d63c9f650.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/43c35bca0ee165a617af78ad0544fc97fbb19322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/4572d7dbdf28c804e97470e2b31eaaed25e765ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/45a7e1926b683f569f4d4d3c35199633602d3e47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/4db7fb0129224a7ada03d3d21f46c30170799a61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/580bf18de339c333b39c29d7e315dfe032f640a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/59bafa80a0ce95176b0b587531b430be8cc795ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/5d63f067f75c1738fe9a32a3bf4f1730d9d30c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/609b24a6635e5203c15bfd3fa22146dbff48dc31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/615ed56104b54b525f58231571cfb7334c100b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/61ab07939dfe4d824b62de5af5994e06fb39c80b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/63db921a938563334d91a83332378e5b248e95a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/67455659bc2c1fa51c77da9088cee9d24b72f2a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/6883cd8f9d747d806a3757b467175572099b4433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/69984c66ec3ff8f31f80d553f711af10baf97d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/6cb26293efe5a9feb4792c3ea7ffa34b13a5d260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/6d65d3d18f1928f441c69d112659c2eebfc84430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/7075af67ebd44564c205ce0813f8c9ac8d236539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/713ff7071c081c054cc8f82cf2b66ddf6b20ba25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/723f2fc9725600dc073d86999d655af4dc34a3f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/75856b35195b5d6ebf167fce420118552353c899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/7b1179b50c8ec49c6f00112fb7715e13defbcd45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/7c4debafbe213b9e21b8b0c27c97ed71a76894b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/7c87214cfb6523775baa66564bd7072e2c2c952e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/821930d274a59d12182e884e0d54590dcc2fdb94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/8c8b926d7d638cd306f1a468f1e0bd21be596ebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/8d4d2350914312921253e4e1446b2ed97a385819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/9501959cbd216eaaa22cff77d5088e0385146c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/96a66268a795698cd4c38292a73e97efdbe25433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/9bb22371fc3b500f10c8641b3cf3217f7f4d0234.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/9fd18a40ded0c97aa32d7625291ab08d5b3ea153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/a2b38dc8b62f44a8fee1a7336f5ba990f389925e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/aa717dc854c35c25f6ef483bb4cf58c7b8e7fc7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/ab99f410d70848b7781bea9da636347d3e97ace1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/b1080d9fd1a7352333f3614813c1608cdd811ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/b2e9ad8ba59015026b553047f2e4ecc0556d6392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/b3c5f2af53b3b254ad0e5c9c65b6ee3b14378318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/b46620199353f1cd03ff9ae42a3e87884ac1a4d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/b7d64d6baf6a7b168c624b9835c3600934f22f55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/bf8b1fd2fe8d228bcaebfad43779d233f082275a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/bf9037a4f2275fc0fd55a8d1561449048949f3f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/c1ad76671127b4219624a381a4b75ecf1b6323e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/c97d8d99ddbe6901681db709ee228f8de3f4c0d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/ce9d93d05c05d4f296ab933233eaaafe1923c206.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/d683a5665299a19030942336f07fdc3d2da247de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/df6acc08b79b6b6eaa9357eaf846649a85b36b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/e2110311d4bea6cf27f1c05b43f37151d50928e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/e73f1b0364c773f5f73ab1c81d1983b1ceb7f9df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/e97b4d6b975021b033613f61bcbd599cf266b709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/e9fb49189b2b5e3567ea2a17b6c64619f9b1e393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/ea17f874466d5f7d59a9f023e51f7b02e90ab737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/ee01ccc72d25cda95cf664ff44b9f8dcbdde6fd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/ee49b2808c37c044d74cc97cc0fef1adaecf523e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f07b652ebc22e991783730817bfe409af5043cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f43c328577bcc46ca8d53f693629f6335cb28ce7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f456b97b69dcd51e5d4485f21e75aebc29cf7431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f6af3c85e51c460e9f2e47eb6b3795b2e1be1908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f6d9f2699ac91edfcc0d45b0e1d8d1491c593f64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/f86ee0cff33f04386968772ece2709d42078fb79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/fe9c3e8d426e4c54bd2a812d1a82c9ddc7630740.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-04 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2019-04/mix/input" + output: "/tmp/cccc-CC-MAIN-2019-04/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2019-09.yaml b/configs/cccc/mixer/CC-MAIN-2019-09.yaml new file mode 100644 index 00000000..f4cc875f --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2019-09.yaml @@ -0,0 +1,754 @@ + +streams: + - name: cccc-CC-MAIN-2019-09 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/08856f420d20866737b526b0f4bf9a3b097e07ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/09e99e0dad7d1577e17cf326e215d3df863500cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/0a6b473636a9a84d8a401802b61f17274a797b53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/0e7d14104cd4987f09beeb8d71db43a159474ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/0fa70bd98099e6406b636caf7fcf9ceb38971585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/109ebe7eb640db263c506806e4c84af4c9b87429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/5be97de6bc0f827524d7fbd5896d7d5abc65a341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/63d6a35a7cf612024623f0ea94f6ff0e40063713.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/737722f26877e3c5998bc772cb4f17ab48ed8a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/738e910d02170584febd893e863835f70ba8cd7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/7fd8b1a48775aa8c21ee83d9b1a33f27eec24b77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479101.30/warc/b4ef2a2b2c01947917fe1bfbc77605b7354caa0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479159.2/warc/466f7032b01690474636e8a2c643d72dda80d01f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479159.2/warc/81b54bbd8f0806f7f690e5034c0e73132f384113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479159.2/warc/a783418ac70ea01e965dc512160f544c6a2aaea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479159.2/warc/aea793538e669bc5a3ab2729a72fe78f7def49f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479159.2/warc/bee61575602ceac8e21ead48d3783f0e76a27a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/38658df5bc6203f3e1687130be31024efe1c8b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/6359c1423b74fd2ce6425fa6d84517a8eb12d3e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/67bd4d16bda386cd5dfc5de5bdb4f923c46c39d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/9ad3f3a2cbadf638487166efed3500c658ed3ca0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/a1d07691d7fa9423b3b62bf91cad4277808adf80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479627.17/warc/c57d06757b2db52ef62226f9ea0744f21ae168c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479729.27/warc/1f006fe3303b7d016b9cd6c0ed65df3411a7fece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479729.27/warc/48613179265a45962d8dc58e1e13ba527b06a84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479729.27/warc/48d979598cd6059fa094527d6d2491c11aa8db85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479729.27/warc/6ecc411caeb93360831cf4c082b4c141b70d5f96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479729.27/warc/a443dd6edae6907deba276d4863cdc5eee64ae3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/0887895e837c9c23cb4f765b0204967622764a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/7119e75aa6f2c3ae0ec07943cfffd6f9cb59f0d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/82032b4cedac16b1023973dc80c403f05684af44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/d66bccf874bb9742bdeab7196e805391a9c03f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/f08c0268d226d67bb9853f24f49cf01bb13b2e52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479838.37/warc/f3d9de10eeda0d7362e82352f085068a61f6128f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/0e990a42d58e6fa408c48556754e96ff44b29107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/41ecd2a3bbecaaaca4f8b863d4903f3d553e7f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/44db7412a77d318898d794030b17c03dccb71d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/9e3cab3afbb496d284fe26b827410e85143c60ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/dcccc8b2b6f71a1d6f0ea3706c854d2add42887d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479885.8/warc/e90fbc7e0ff6ba3d954d7daf94d883da2f842731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479967.16/warc/1d7c549661b83f8933fe20f3fa2628e7d7a96971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479967.16/warc/2169bc36fb6087d526aafa63bba3d24512e6c3de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479967.16/warc/3a546688aba53c555783eb54a5a6cca6e8fe2226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479967.16/warc/3dd5d28bbc2282237d9fc0ec8383c4d97a1097f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247479967.16/warc/8682a640acb652eac53afc15ae10964822d134b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/0b076053100b211a745f37c4ae4d933de4af9845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/4aed4788efb83e61261fac668663aab5edc12d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/5196503ab75f9c4873ac284f518a5dfe9047e6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/c7cbd302f54f90b36bced8ca659f73a3a61dbbc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/dff314866435776d39ca4da332ae63e8725315c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480240.25/warc/f8b883887500467deaa5f8fe77f7e3a5bb4f47b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480272.15/warc/2fd4e0538562ccf339db7cafd509aa0d5ac33fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480272.15/warc/a387585032db0a7b813f7a2cfd7a4da8261b1149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480272.15/warc/b1bb4dc20a13da398f3a147b5d5e40b101fdd499.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480272.15/warc/b4abbf9f69344b094544f19adcad23165f0e0773.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480272.15/warc/b64637136238274808f2b619295f74b5b34fbb9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/4fca3cfcb7a8c995ff9af35c87e502a2a3d9f846.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/502710ad218d6a9db73f71060d85e1c5bc1c60e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/68bda96569a6079665005e03ee296aa28d9aac55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/8ac7c71fbed23a2baf66d77efe3119fca966498a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/92ccfff20be48f61e0eb17a05dbda72121bc1346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480472.38/warc/ae9e698c5917e51158fc462a26823054f5a4f2f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/2b9fd64752a0f91cdcf6554653c908912817e7cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/71e8d6ef452e445bf603ed61b4162a8e87a0a424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/7ef80973c57abc0609617ff48ec23b10fef3433f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/7fdaac35d7a19a9a8dfd01ea13534bc5c03548fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/9587c6641f836ac9e027a3f49e46cdb61148b4f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480622.9/warc/de4c24defebb8685a4955f25232be55955201195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480905.29/warc/22f7d83089c9776cfdc2a6e54f6fdfcc056c9cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480905.29/warc/83e90a6edc0ff7798f671b9dbd7c42ebfe8b62db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480905.29/warc/9d71347781083d322bf342c9158127b60bb01bfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480905.29/warc/cb3c1b2b7e90ba65cacc8406649d90d5abcb026c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247480905.29/warc/ffbe3802e5a7c835010b8c7939e1950d1d2e0127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/7345304f51c213d628234fc87571d9e54a6e433f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/a191cc641a4198049c48ce12a437845f57d02cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/a389ace9b75b34c4e749395f8a11d046fb7af8af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/c2461216a70e9e4379083cdfd18c27e3f9a364d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/c7a0eae37634125ddc66a872604635a8b92bea8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481111.41/warc/d20c9c3c6085c39cb6f46379f5c950b338188481.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481122.31/warc/2affcc7a5b6e3d7006bc17f15130523eba3a79dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481122.31/warc/3d1d99ce961ebf9ba2b594f2d5d799aaf35e5df0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481122.31/warc/483e4935a71137d897df682f8ae430fbf50972a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481122.31/warc/56c0051433d22e1ef474d716bdc01ba4ef5a0799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481122.31/warc/ed43ce34db1f79ff5a798e45771de3da08ee6977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/3fd44bca75524ab6a4a9e8e3f742c55522d8d769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/5ce375b5d7b88a234be7365092b64541c0fcecbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/671c8f9f0a431e3d7554216d9d4cc6472f9f20ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/6b8211ca3740e1ae7fdcaa685352227ceda19fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/98fb731844080bb51ec102669e1e947e6e0b991c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481249.5/warc/e150f6b6560fdc1bb6f74ed2142741596d540eff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/126f811326f9cc98e8ab8da550a25f6b4f59f337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/7c79aecec29de9f0ecd6791a84a57f0cd819b3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/7fc1a9ae3493cdc6bbd6acf1e579c80c91d5f79f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/e0112af9808ac49a63ef180d8fba3d179a5ba0e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/e303312073d4d418f36a564461262852f28bc74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481428.19/warc/ffb02531a83ad4a4cd4fcfbf06f68d5b9d11aa0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481612.36/warc/29c5c3d02d13f99dcf3967b3c439498977b053bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481612.36/warc/32d925fd4b56aefaa3c8e405ee07f8e406c2f9ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481612.36/warc/3f8471f0c78899a613d28f18a2cb0f19ef6af8b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481612.36/warc/da074ea93d43ce148ce68f44fc7e147eb1963887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481612.36/warc/e76b94a04bda07d83c125261efec21768705cf96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/3ec7499ce8b5c3a4e845b46347579227543e9b44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/49a91f11e88f94a0df32f67954cfdb2e959a1952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/af1a235fcb520bffae1fc479a08ca5fcd75fcca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/d063d6850d8b3c685c5f9b1e1549fc81eebed106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/e79854dc9c8e533599a75af51b08cc7f9c4b2c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481624.10/warc/fa3e07d7421c044c1850ac33f0576d187d257313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481766.50/warc/2fe7484ec19789f66dd6e95db46fa47815c644cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481766.50/warc/81d7133d7bea17cc7e484b61d27957800e46b9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481766.50/warc/85d8d1ceb0422be77ddb2ba919243ce59d7925c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481766.50/warc/d0a5f94ea5eb7dd06cd39414fb48b46d59689e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481766.50/warc/f460a93987e75386f3083d1cc7ab52402f8f8ded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/299771c262a91e029d361b91b2c1bba65d13024f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/4daab928702ed51fbe2231505342174cf61eeada.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/825efd8247c5ee89019cf3ce33d78b65a00abfd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/b76423e28ea49bc38c9cec58744f83abc8baa675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/d994847c8a2466d12aec9fbf589ac737e10571e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481832.13/warc/dd27487b25202fc00655f44bd3ffeb474a1631a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/05341d3eadafbf0b4953fd716530b646bd239fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/0989a48ace8fcba7f6b9d78c35f03971e62043f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/1fb590a5c444ba52066dde1241ac2f96b138a225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/5ef2e776ce6d831f3daefbf500d72268f31d24eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/8f26a4abd5f6ba66862dd3da1de1b48815419618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481992.39/warc/903c72ec8461e156838b6e2f0b1537d937685a91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481994.42/warc/17f887aa5c7169804dd7491fb3dcfb3168918fce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481994.42/warc/4d9be9801de2e836bb11141c3bf242404dd382b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481994.42/warc/9773fbdf2c26d43e368624fea32c09b477486198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481994.42/warc/cb204b5d0cc820cbf38d85cdb170acea3d60cc4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247481994.42/warc/d0a9d9808dde92a6422ade7101e34eed75002333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/41b5f059ff4d9de7a6cd611e9b31a9d053d7d27a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/900e2077473f158b442c47505b6882dc2c2107ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/9663332e19bfa4b999f3381ee6a02b71f96a3483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/c10747a42f2a333bbddef84a3b429d99d2551adb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/c9d577d02677a750c9e2443cc79e08abb0780ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482186.20/warc/dbc642b6b3996ed109c9f6fb57d52bfd1e4a25f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482347.44/warc/3cecbc9466569318648071c5b50e3d17814cfe2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482347.44/warc/5120702d04d260a390fdf93b7efb6c2e6afa230d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482347.44/warc/b792e8cb2191c1342aa89d8193b5abbabec3908f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482347.44/warc/bbb3c240a2d395b5bb920dd83b95dda58557998c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482347.44/warc/f653f920145dccf0d892592cd0b90dfe7cd522f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/3bef386a22a4e00940e65dd37e4709d5b1640742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/46046e7a9c531f3036fe6c742b8f611fac05d5c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/6a70d2ed3110d935805180aa5203e05d3c4aa529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/6d2c16d592a91aad7bd54c9b373b18caee5aff9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/c9980b717307efa341de884f06e68eb6ab26fa97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482478.14/warc/f66340ff6e0b95d0755b92611bf3fe01be75e622.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/02cf83ec82fa18a07d759732e0be26c14db6200a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/05ebe7d153782a3ba8433dd28d49cda2c5fa55f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/0ca13989a57ace7d3ee5fce868506986171e4ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/20a09ecf8e865cee8bd07edee2c94ab28573dd9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/a3323940463c6983ac82959a2b63912c508a484c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247482788.21/warc/fa8fcb5f8ed24280a5d2e9393a6dfc07ae943e53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247483873.51/warc/3a9939198ff506246723c42bd7e9b89fb855e607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247483873.51/warc/6559adeffa13d856f50b5ccfdeedf2f3cb7d1563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247483873.51/warc/92e6ff7927a98565dcffe721e6a563d4317d9a91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247483873.51/warc/d698005818945dbdb8ea3b73cb96cac9f2848865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247483873.51/warc/ec687b51b1a27a1f55ee1a2ebbe5d4757a4503ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/3b51108fa005b285fb2170cd0d835da019106817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/3b51f093322969b42b97b944e572556ed610c252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/5ffe061d0f27f99917301628b0ecf9716e460ffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/7b9f18fb912b404ef76274d4168df7c439908f19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/89ed15fb12e307d3178f96992bbd7dcc4889a5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484020.33/warc/bc109553330c4afbfec6a8cfcef1d041fe86a806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484648.28/warc/6f178522853e9c7eb370015be532be169e35c911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484648.28/warc/7c6385efaeb93a15da70178faa3b314f4ed7cae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484648.28/warc/a033f4f897fb88efd843396eacaafbffb868a1c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484648.28/warc/cacace9e8580b137eafd1d406faf8248b05e0c33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484648.28/warc/cbca28bc7c724c30e11f238ca4a0d3e6b9441404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/26887df57be91a991307b7c8c0b5c90c61f149b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/4defc9cae5707043bafd8da5156197e219a2894c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/57b58f89ae3e30a135d2a5b0acfb01a05687db9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/86badae20b7c470d5c1230766f1b35c274761699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/be4eea8925212217d1e3d41ba039c4ce57e00fa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484689.3/warc/f376f1d9112019c341ebdba4c384b06366ad8e68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/6fef90ec9154e6324fcd67c2158c49d421516f3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/9d88d70847ac8d7edcf9a457f7d440511e2c8315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/a5f957b34195029cae6207ef9b427d61aab25364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/a97594c42ed7bb5af928189481dfb6f9bc238ebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/b1fdbfb22e8592ed765316ac5e6ab15503153364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484772.43/warc/f287cd87da67cd291cde83f4fbbc6f9c30efab9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484928.52/warc/0350d0e26ac83c0458f499f1fbbffdf5e7fc0e59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484928.52/warc/5d65deb3a9c5cb3ce2170613af44ad4453af2155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484928.52/warc/bf178f9e8a89e307f3d9dbf4f7c74a7d7ee6dd47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484928.52/warc/d5a4d24ab742c8b59ec8665c7a45d6083000f1d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247484928.52/warc/fca34567aa40bdacd8f05fd14b778362346bb751.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/0aba7403b0c2cc1762a52c2e47412665beba9793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/21af8d13f60c1bf92dad7f747e65282022d9f912.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/9c875a5d3d994e8466b062f390d3e58b71182e7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/c1095599751f5d940250d2726876ab11afc6c3c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/d8d0a63b66c0bb58c965518691619ee895a406e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486480.6/warc/ea474d6dd516359f6dbb3a4248ccb7575f7df7fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486936.35/warc/42afa38d004fa5bad10d13b55850b31eae7d98fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486936.35/warc/6b85a70c1caedcb8723a919785039a1c27803a33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486936.35/warc/9e2db4905633c7256a412343153e30a6c648f16e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486936.35/warc/d4d5a61283014d2b9a756dc614cf07e1e5d00657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247486936.35/warc/e279da9c8d5f52fed4ecb545d7dc51dd7a0f95bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/0ff37f46da55e05fe9d225c36cb7c93136030196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/5a54317c9ebb9d71d98af86ea569876934d8f0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/608a6adfb543eda17b024bb48fc8d3989b7cd291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/c40086c611d60f57fcf810638849262b0778672f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/c94450c3253411c7f667e94b9428ac3545350689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487595.4/warc/c9cc340280502cb7558379937f11169ccdc368fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/3b220e2a2d554fd555d071061c5c6047fa622b4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/406be567873a8e9e9bbe8073359a3dfb730df2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/4f5d10c3c80ca503aca556cc9f96e1054ac0e6a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/921fd41056f0dcbb0ef65a306b279eabe8728a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/a58a87bd2a98926243e7d2b38db0ac070973aa07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247487624.32/warc/ac1eac123eb3427123529fb6726dc2321c2cf20f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488374.18/warc/44fe84cfb086d90f2c6b95cba8c0b3b3a6c676f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488374.18/warc/956d4b12df0e5e0b05dcdd311ce661cb3836e864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488374.18/warc/af8fb653b7eec4502f181f73725bffe61881fdff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488374.18/warc/b77668a169698fa5ed761e1b0b48a8fec631f396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488374.18/warc/be1d670ab263098e173e70a556d5c464eb45c8d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/0e75ec12b3f1857e07dd0656e2d5edcf8b4a0a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/53a6a0d33339888b29a71d49b189aeb9840f862e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/9eda8e2ec7d387e1cec51ab8d3ab4a8e191e5344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/a722f3b40253db84016b0703a79d24c096c32522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/b16d6651d512b34a7e40b98e182fd9758ddba557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247488490.40/warc/be7d85c286472cd2b3724f06f4975186e15a177d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489282.7/warc/51033ff77da26e7b173df2ed7d86e213344a4c64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489282.7/warc/6c0f483b76c52958484487b4c79d97a388bd33b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489282.7/warc/d3982f84fe3c28f6e830e77f55a852c8636386ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489282.7/warc/e6e2ab48fc009318c467adad5dd1497e03ffe5ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489282.7/warc/f50d6e90609625bf33a5da0bbf92cbdde2c3a656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/1b03194a3359bcdf60ccf74dd4b6728dde0bac6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/2498a36990e4540d15c185f2b6abf3a3821944dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/308240a27c422c5cd51ecf8ffd1281c94da0f6d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/339a61e38bb2c12246c9aba4dbfac8c08ddcac41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/3dd09c9e64ae3cbf906f2b986dbbac018cb55740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489304.34/warc/647b73952f956c0096b2e1e1b3875756e0e630e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/0af0db2905d020a11058d0740b73538c784d6925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/0c020a4347c92967dbb148483fedb4980de001f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/2a8e7c692613fb0bd0040189e11a9e9822e8b191.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/35e21bebfe65d9718d0217ce44ef92012505ad30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/87d3bff3ed7e3e868c54f9cc5a39e108d2a662bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489343.24/warc/f8593884cc72c6aa7f0d289ddba381d1eb3d1aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489425.55/warc/2b1d2d0178856452f28c524dd366addabff48688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489425.55/warc/4afb21e430085cd46a2fa4c988332dd2da23de9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489425.55/warc/7c6c824a72710f37cfce1ad9ba6c37e200da6c9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489425.55/warc/8138cf6a5df0e86fc1f61f863994adb257e6e52a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489425.55/warc/cf22b7bf2cf4f22a899a32dd02687eb105bac673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/13db915da9372e2da8db1f89f9ca5be434f4ff08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/3930e016ffe50058b508056c0ef145f718852829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/4ed25c508bc2b7267684fe6992c71c654dad3662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/50bc92acebcbd128a83e6e8dc92bf48b97d23201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/6e4f52ad9a8cf8bce8cc4e3052d4cb0a31d02f1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489729.11/warc/b5b77cd9b1200864e833027df9a0f7bc7790cfe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489933.47/warc/36f9a6a66b2d97bdadf6f994860e89165125fb3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489933.47/warc/494aaa6a5378778fa4944fbc41a84214a4c14e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489933.47/warc/807978f4444d504088dd707281a7200596098283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489933.47/warc/cb5f27c10ad77fbf2a411a14a0f8743ea33d313e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247489933.47/warc/f74e75cb3494c07f868e54fa2c5a85ebc4b72f35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/17436521c3661334262345caecfebe95fc6bc20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/26252242457bfeb4fa7327485291d92666caa504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/48262daf9d11b16e88f2a85318d57c5ddbe683be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/a5dea7a05fad9eec87676746aa788a9eb8b9846f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/b991d938c8d9bcb3265084e760208235054f11a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490107.12/warc/c4f1ee07be73fd11d11f4676cb83a2da48be2fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/3a70a0ca522e3ca03a4dcedffe302de0a74071a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/89b96e96abd8cef76ee3a41f8512493d0c558597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/9dc126e74e848f6837a8533b52f69fa48f252743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/b0eeb2303a141237a87d9c7be6f8c54673cbe00f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/fe645459ccc5781cbcf106ad74f0e05bfdff7841.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490225.49/warc/ff0fb4ee5075b9a551bc9f471d57a7c42acd54d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490806.45/warc/1b67e65ba08705f71a28b3142ebc002399ff3ce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490806.45/warc/36bea5676d1b0c6e91e4774c9a6c9b3412326c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490806.45/warc/89f03f8cfec8c6628953a55d820b2ecb51e135e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490806.45/warc/b810cdb4f3f2af5a5927b1afe177e0de88224c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247490806.45/warc/e77c0c25c4f0447bff1197942c4d7fc18cdc274d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/422600a39fd9ac10b6036004200bbe5e6766a03a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/47e2d890104cbfe3f09af51f4bb3413b9f99b5ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/6ac899eb0429abd5d2c75553845fcec9ddedc577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/8bfad09820345d9ff7d8f774f0c6a53503c33f97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/bab72018440ff9a5d657722aa74fae0fcd8b8c0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247491141.23/warc/e85a33d0dc7a80ba97b4a3b8b63eee9d833e3f67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247492825.22/warc/03583f2998b2f217848b86085c0ad194ea41134f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247492825.22/warc/1c072af74ab9a943a42d8238f9c3ca256688fb07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247492825.22/warc/2293aedbdf71e74154048f12ce737a3dc008bcd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247492825.22/warc/5f3bc10fb09858e0777b73eeab5455b25d4ae14f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247492825.22/warc/6b575bf26ca95d15c6a25c056b6eb82c661885d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/038bc50d14944853cdfd4067a698bd3abe5eac23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/0b301df2e98ee4f11960f7609540dc095b834def.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/18d921297454fee13175cd774eda43a8b2f72a66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/388104144f8d955c9ac8b67e7f93cbab7e4664a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/c22e6af46c12dd97359e53ce20d19c7c6a427e42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247493803.26/warc/f499f7f394e924953b7ceb0da3d543786fb20106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/1c031a3ba36f2fee62eed50655b021df731d4d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/291c4a0c77acea376a194eb72be06ae8b4e40853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/59ca1008a9d15ef9223c7de814fb0363f5be5498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/71d0f0050d0961070f840e033aee58aba1f70075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/e53fbcc202e6d209da70d68ed85ee1d12a8a9da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494125.62/warc/e86193e2db3e3dbed08558f99e238a24ada065ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494424.70/warc/27d86006d8e9eac9dad6e37971b4b81ec8cdeffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494424.70/warc/400598ce804c0c1af616bfa188c31afca6ed9dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494424.70/warc/778cc85919c32c169aecacc954282f297b1ac167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494424.70/warc/90f80c00b6b9ea920a62bc87a74a0928d8472beb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494424.70/warc/ee67f7312fa0c466666a6b887c643b613d0be6b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/245efc9a4bd8c8bfa692a9032455bbec803dd9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/721575c432b7df04509bfc70052bb5e5c222255f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/8928e96011c83ec18e64e55b0087721b7e9f2410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/cfcedbab4c30b781deecd3e2581bc695ad41db34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/d38cb1a4911ddd88be461a4577054494428915fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494449.56/warc/f4cd0955b9ac1c88abdce1339eec3333e5164bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494485.54/warc/6b925879f588dd3d0b9779835f33132ac7cc1298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494485.54/warc/7689807b54b920951ce9db949b3bd7449b8fa1ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494485.54/warc/7a9c2cf1b3a70beb634381821f1fa83b42a166c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494485.54/warc/7dfb317796795115dc63f1fc3df8b02a5393d3c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494485.54/warc/c42b499453ae86514fdf28f306b92df221e108c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/34b9fb92ff7bfbbc8687762ccacd9c84ec8e9e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/439db199321b993dc0f65f36596f375f3ff91ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/4f0f4a1c2feb2a46bba7917f53ab9bfdff690f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/b9dce0baa78fcd9ac6460a073faa61ef272157c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/e112631403e140cfa342c10db59ba3071994c0f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494694.1/warc/f5d2831c9a6aaabbbdd9f0e7f525f041f09dce16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/2c00ef09aed09313583a8fb1ce36d2c81e6d7691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/31f0128711dc7f517962342ed5f91b7aa9a2d259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/3576875ed655085047b58de3005f1be47abfd2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/6b0227e943350a8199a4613c74396a537d2c0409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/723af445a66343d6077bb7d6fd53556da4ca947f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247494741.0/warc/c110d1ddd5617d4c139881a0e2be3a1ca134d25c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495001.53/warc/0cf2d689c49c64bb9a623d1586bed03659448c7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495001.53/warc/3c998f7175c554342c6e5c05e2a68b17b457c286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495001.53/warc/426d7868b14fe68339da1f2016622031cbc60d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495001.53/warc/9bf1c2a7c88944981c61d1988b243ca80431c89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495001.53/warc/a985e12af7670cfce0df80165c5c191a1a1640ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/0c33a2a624fd11b562d9f5ea28b3e7bb8617e065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/3f6951804279dd13f45d8d08ebab01228d1e9a36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/63808b948e7a728fa289e20466bc3cb52385582a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/7cc9b28fa6c08ab23958460e79bf0c55d023582f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/a5e378aa7017f310d9b6f54f08743975c4c7b1b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495147.61/warc/b6dde847134b7cf1c475774c89dda039f8aa94cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495367.60/warc/36994326cb951d66f0c7624417f57d1809186d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495367.60/warc/4aa70946ee61f2ca58749791bb2cd53d1b942f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495367.60/warc/9937e724d6a117e15073eb4f5219ebd17cde27ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495367.60/warc/adbbef8decd60573f58d37b116faa3be9dedac1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247495367.60/warc/d006822cbc39f43f3ef6ba4cbf4d1b34195c74ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/6874fb4f1546a052177483c3ca81b85f294315d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/7fcf66f84294d608f806d8d5bf1699c255b6e204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/a1d10cc1eee4688b7757d179aaac59059e514430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/a8199cc107e70d32b20b2b2b0d3e80feacb90d41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/c200d8d33497198d49a4c967ea2a65d1c67ecfa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496080.57/warc/f166c57aac1d9128f59806ce2d1d169bcc381b50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/004643a6348e0bb8b4f7eae06f4e9ab9a7b27eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/064779375d5b3bbfa3c1e64a31b2ec68c5afb703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/83f85777f1c0a89ef2e8c03770e28049f2c75435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/8f693931b1ab1863ca24f0fb3ba5190cc1715056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/a6a12e259c5265455afb38997e6489ad50a95745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496694.82/warc/b1b9470544cf0e28e676776f738cf061edeeabc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496855.63/warc/24c73de73065cd3565363e45105b65a1697245a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496855.63/warc/32a8d561dc79e544a4d2738d0ae354e11727cce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496855.63/warc/38ccb228f9c8380aca144209c089b8a4a4d35004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496855.63/warc/52c07f3df997ac00fa24a2e516199e7b654bd885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247496855.63/warc/b0d2798d8a4e3330960c46e1c328a1d3cf051c78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/32e6eca6a701c1d1b9f1dba9c8a5f5db703e1d3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/658757e03879a4ce5baaa98d42154e344aba77f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/d060e016b909717e6e9ec36d20d84965bf71e6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/db9aaf5055ed943c177a9edc78fdc5b772a03a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/ebb9b6a2f9b97c1da7c2b5093f35dd48d3b1a203.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247497858.46/warc/ec1fabe02fc1fb97a02acde614a03f51486a66aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247499009.48/warc/24e38dff3e5a956ad8b4d3a6babd6501a3cc9b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247499009.48/warc/35fefd58cbd9628bf7e8e8eb3b15d74e6830ef67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247499009.48/warc/411c79282ff928190868c6ba9534a307709095be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247499009.48/warc/55620d9f881711b25ae3a370e676778bf1761482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247499009.48/warc/c4cc5388f5662dadfdce6f86fefce8f06d540bc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/054fac0eb65a21502a5d8bf2d6644e64496461b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/193edc65f5f67c7382cbbe723ef0880ce760b6ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/47e00032701fbd0847579e89e9b670b784b3c794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/6ebc0494171d476e548b2c31c78b248bf8ff74b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/7efb93c6c84b2f0fadae554a324ff58c5cfb808b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247500089.84/warc/9ac6c9fa7dc167ebfb5ca5541005e46634798a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/0fb23979236ae5e5faed5403ecd59ac608518440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/1a6814f00583768ac144dd263a95233f7c383447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/69671cd2f01edf4731608de912992a694a681358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/721a6d345cfc9bb370069e8c90ab08b4e094d88a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/9ca8ac5cc70bfd2630f8b31a9e18d9bfefdea33e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503249.58/warc/d47afca3b8d1e1f8758b9a7ac98abb713ade72c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503844.68/warc/4b15b3d8309ab1ec32485004463a9e1dcd2bd147.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503844.68/warc/680324b413f0f4e612bfa0b88232927b6f483202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503844.68/warc/907e8641356259fba0c380d98857ad7d59e1f801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503844.68/warc/9bbc509357593bd543cd13940c764e58f495b9ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247503844.68/warc/f16c51fcba316bda25b9e72e65f823d4d4a16afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/0cc184b4ccddf1a8d3b6d9520eb2dd575a1c3c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/38efe515513c4a83dc1b5a5a55fe192960a3aa91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/5e5f081fbd9d70c0a8659d7673ccb1bfcde848df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/73e16586337185b2f960d65ba5f1d9a951db0a3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/7c8725f64d4eefef981f9bf6f638f39b0ba8550f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504594.59/warc/f70aa6bc661904c8b6d2ee45dc56328eaabe3479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504790.66/warc/00abf931b0513ddf83e2fc945fce6c55a6f8ad4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504790.66/warc/06a646d48f151b0c99eabb00c100ed13c58ce6d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504790.66/warc/23b052a9fd9e6210fee0632e78d5c801aba535c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504790.66/warc/ba4e6c575b312b30f71fc40e264207a9b5108546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247504790.66/warc/f157b8a4314aece51effbc88152edf5f201f0ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/0186906da424061c876cc6b3913dc54b4d3a394e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/0587df4cffacb72f44b1b45c19f13fd5ce76ae9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/2bd0e6ce59680ca225558df825e2f4f99a15b638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/7687cf7c159ced507a248ec6524b6201eaf4e842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/8e78d383b93721d7593328cb913067233c5fc9ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247505838.65/warc/beb6023807b37e2abfa81ba6839f0ffd6ddfa1b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/02133efd596caee210c502d63045011613be0dc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/06a5942aa564ffb8af04e61f42ebbd292e0f3fe7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/159b427070cf3b4c134beac9ec8250ace4098fbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/3cbf8e25b7d199669592ef2d0e3f211af9ae540d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/9f10afa20316e340307e532377c1c681ecf10489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247506094.64/warc/e50f43e4f9bf71c0543452ec23bf211dc06294a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247508363.74/warc/36dcd6351b628ea8a1e83983439c7f5ecca3cde5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247508363.74/warc/53eff403a9c874a496b119663f077f74b684aa97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247508363.74/warc/85cb2cae08cea16c2c6413bb05de840feaaeb605.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247508363.74/warc/87e206bc1c2f9a37f4d62d1cfabc27c72416552e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247508363.74/warc/8ee8c0703764cb234ebe437943a2da673cb26923.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/481d9c9cc4de76291c19ce4c1ee0137fc5166b92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/4d2efa67af5bcd8936cc28eb18d33a4d5a98b95d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/687eb9b8866844c564a94f4261103c014a2510df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/7cb5f91b633c6e0ca36bc5ced054ac67e4fa3b44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/e404731367af584f53984bc38c23e09795ec1031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511174.69/warc/fceb4c9cca4f32b80052c106bd01310828346b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511573.67/warc/0561971994477f4b3397fb4ef5633178e6090983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511573.67/warc/3ce129a731cce83b45de8feb77285baa6f7197ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511573.67/warc/6296affad6ea46f15de7fb45fc061c17533ff629.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511573.67/warc/8a363112c7b51e4a664a9b340904ae1249e8df03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247511573.67/warc/f44aa029557cdbb15918094eeb57c021ea359ee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/0ef7124fc179f1cc74d96d0533b94c5e29b70638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/1377ee93b3207f9c5e303f6f836340d63c4a7082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/4d87d1b7a2a0fe07b3f3af7ab81235a943323dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/8b1ac45d1ef489c0e9c9baafb0ecbf9345ec71ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/93e4bb19787aa9fb4b6e7e4178975c355fb50d50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247512461.73/warc/d90a4b8c8e32347eb7ef48a8c449dcf94d6b83aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/02c69549b41b9366c34fe008e8e9734728f28e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/0402daa545bf783475c7a553a6a6a2ca93e90f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/55f33d99c2206ac3b2a41980a55d581d0093f922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/96769968bcb803a24112453436b411a7d889cb4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/a38bd96cc3b70e5dfeafa4df89d71f827c6c443f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513222.88/warc/b16dd61bf0ddd3ddbe9550a3118b89a87f810980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513661.77/warc/1db65a415dd21d782fa996b95d01a6fd951fa734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513661.77/warc/31bef2df4a93f2c6fbf63e86757b6de8c878965b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513661.77/warc/48aa83fd8b83f8e69ca5ea2b49ed89790f9d7a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513661.77/warc/7c8cafc69f1ba971b13dcff0732b05877c2fac81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247513661.77/warc/d0872c46b9e1e74908bebb30a6da2ec8585ee728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/24ce34477416f224275c24780405fdf763e280dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/30e3c2a67a79ee67a2ea4861671ec3c1f6754ade.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/9fde88d53c10577cc751bc3d79fbe719e63d0dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/a0fc7f1f63c4707685ab096f4f8f9977e1ecae5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/a9bd9a6ac879db4e77658e986bcb7c187d10dff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247514804.76/warc/af22cc4fb2cad02f4289037e24e0ecb483740d92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247515149.92/warc/2f49d11993584e2536e6422f9acbaa6f96cdacd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247515149.92/warc/56597ffc0227bf2334e57a1dfb89a9aa74a119bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247515149.92/warc/70abe06e9c935a631b7c2852425bf5a36723dce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247515149.92/warc/798a8380b650fb1a0d0980da1334ba1e467615bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247515149.92/warc/f34c6dc6ec2fef410d1e191ad80baa1c5b1377a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/16fa11c6db576ae77c80c676474168301d4decbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/2369cba0d86628caa13871339df6c7d6ae9fe32a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/26dbcaeab5f19f0a9f7a27543502248dcaaa15f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/6095698b0980f23669b32fbbe502fe6789cd1dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/6339c7e2ecb0f30f5f493896f405669289c42bae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247517815.83/warc/a672068e0f80362ffdefbab063893aaacf005356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/406692a4329081420415486a8e97bf3ed4155e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/42ba362876790cb86d2de093d30934476b7a3d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/abeef11b77b5f6e3d37fa2dd6dfdae979be54917.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/bf5aa0a734cf967af46ba48432bc5bb450adbac2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/f69e44a616b24ae9f8ebf6d132bf4e45986b00e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518425.87/warc/fe2d9b549c9d437d210cf7458458e37da5d727c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518497.90/warc/4f2a378a47e336ddc29cba551cc31fba8245f7bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518497.90/warc/5ac3fa358b4af8b1b9b699d5775e88ef475199c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518497.90/warc/b28ed95c74a171cee0bd56d53084dd8f350d9936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518497.90/warc/ba337ca9d6242b6638980e3d0a9e1c9773821ed6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247518497.90/warc/d1d0a15918184ecd852d8ca2fc3207e1acbc4ac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/0eae0745ecc4a50032d4334dc458fa5610785d14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/5c54f6989d426b08ceb4b3a9227da3773083ab2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/bb22dca858f18c89b32a0628493b2ac34e35c2cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/c7dc9c3bb4b809dca00fd69f37816ec91859a27c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/cfeda18926e2a2a5efcffa6524d9fb00a0c5ccdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247522457.72/warc/e45cb2674abf47633f8a86beb1163289771a574f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247526282.78/warc/2a56ea80b139dc48019db1183c452c24dfbe6664.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247526282.78/warc/a0f911fc024016ed6453e615e3a30dc1971e5583.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247526282.78/warc/bec90e3edf18f4d48e879261d9ddb9bbd2bb900a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247526282.78/warc/e70ba1a2231ee3fef3377ac47d4cd5d187040e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550247526282.78/warc/f12d450ee764f1bdd0016c974b3801040850726d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/10da2077e088239c73a8fdaef4ad1ad3d65d2b2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/5e57ee8fb78f4379f56ed8cb1768fb7013bf75c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/6f1a0449a0c56b62bbc7db344a308668656dcc0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/82de7474e292af35e94131e3acadeddf01062cd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/831ce1ce833e26cab78aa3caef9e9f411beedfd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249406966.99/warc/98c48c957451d5aff82108865f86429097a9e2b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/5ac1d55cd0abcdf1c23bc656ed6cd9aa2357c51e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/79fdd72960cf7aab9513133ab23a93efa3693277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/a13954fa2cc2c4c72efaf8df2643e86d8b7e32b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/a1848cc378aa8c02c5c179d9a589556da8975113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/deeceb739399c6864eae2ec1ca0a4fba3dcd5d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249414450.79/warc/f837f9c8b00edda52a8c75b5ef4999cbbb692f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249434065.81/warc/036b7c5d881918759942d01eee2844a0199f059f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249434065.81/warc/6be1ee2b9d51ffade1ade1fb0235cacccc89cbce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249434065.81/warc/99ae06e27d2bfd3c78cc09240564f712107abd8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249434065.81/warc/b30366c35d3dcc60a788eb9bcc02cf6a87c31420.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249434065.81/warc/b4a31fd3d67d68b6a531233eff8fedf76f6086cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/386fc3329e0083246741bdd0230bed2a441ae055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/3b83814b25730ba94e26432501345436317765f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/4523ef490a28b94c306c76845716a8e3c47866d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/68102ee6cc23c3b4b085fcf4b3870fb6f80db3e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/6bb4b4841620b0130e4a99ed13de14fc23779456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249468313.97/warc/cc99b124e7cc4beb49075857d1e805604e73a5bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249490870.89/warc/198089e525e832ca4cec323aaa092aa00590d73f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249490870.89/warc/1b3c0de9ad7a8f1e72e3e9e3b6224cdd01e6e041.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249490870.89/warc/1cac41e3a0870abd228819629dddb863054fefd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249490870.89/warc/289e7abaea952419937fd0342a19858021a4a118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249490870.89/warc/fb694101c22e8ac264d0f59ff15b5321055e2dc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/667842e19e823e92dfe3c3ae03a1d801ca383175.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/8820bece9ad0d48f06ca455930c7d8b4ecb1eb8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/a213c7be37067176302ae4b83d14a4ddb53ae690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/b6d0738325a70697e1b5a4a021ef203bfc1657cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/efb38fab1c1c3a905ded6ce1bc28ee909af19c9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249495888.71/warc/fd847b84b36be605e76d8dca80a8d8b4f49acbf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/08014ea2fd58002ef8a03c74edc24f9836776bb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/54c80548265b70beebc5e0bad2167081508a065c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/6c1b24043b9142494c2bda9521b79399a89f5d0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/bc06ed700af2118f93cb8b01074642fac78d05b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/c4081d14865c7f39edc842401536a35a9a75f802.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249500704.80/warc/c8baec598cc938e1a7a9d5e48f923d28d4af7d26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249501174.94/warc/03bec7ad860472791c2e5a7ea194c7c80ab8f95b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249501174.94/warc/1163d9f88a8d51b227a307a91cadc011b69fac6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249501174.94/warc/896f45b0f736c2ad53fa22b5a3d9dcd7131e194d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249501174.94/warc/a3edee2caf9c785affce32bc13fa585b59538bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249501174.94/warc/b7082fbfe232f1e5d7fbd3f91b54734d410fb89c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/166adf93b6a7df620a9cd220b522de191ad434cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/6a39f82c86b01b38ebc40338d06561fd7ecd37ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/80d42245d78049be67001c7866ad0d2347d2bc95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/9963f2186750c5863f5e88412aa18deff5a2308e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/b55bf0679ed905878e1358bbeab3920f0ab2f79a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249504746.91/warc/fa16ece6d83434e4c580d45873dc90818d26a048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249508792.98/warc/3016d87c1bc4376789213ab4a462bf9dfb16fa08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249508792.98/warc/50f230402d19dec25e68f8d509139a3fd2bc5b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249508792.98/warc/60ba1a960b5bfdcfb2c40f235276ab61458c663d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249508792.98/warc/8d940ded74cdd1290b42bf8c7120e4c7f8eec0bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249508792.98/warc/dd03a573f1c4e1191e21215e69cadbcb1495ccaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/024233263a3b79d43185ac68e99b625a245213ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/060758e782dc6ca92f0f3b053c062cdca556fd83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/ed5f7e6b646662ec457f8751cd3611d3496999fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/f527d355cfd7d60af846d157ffa7e1156f606774.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/f5be5074af9805e2072f5eac25f8ee263e72adb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249530087.75/warc/f784627fc55dfa3617f3dccc019d5b24c0f44c29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/14ac22d0c5f3462bbbd7b2ab160b1e01df06ec6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/5d94ede847ff6f194fa53842d2fc5c1965175073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/69e0bb8733ed322beaf008869db084b64e0d0ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/87d16df81a2a6ad46759cf9c6303d1a48b9cb50c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/bf676fd30d480c5c580d3b219ba5f8c949a0010d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249550830.96/warc/cacf49088cb0d4bf9beca6005f105cf16b224279.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249556231.85/warc/469d34a86559d9ebe036ac1c0e63bf10abf4d679.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249556231.85/warc/751c45eee06fc4169b3d57e39ecbbfecd39ba5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249556231.85/warc/aab8892ce8010ac52a94c9376923bc201faaae9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249556231.85/warc/dd37e306e32be322b9371f6692f191e16d2410f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249556231.85/warc/e7dc6864d55d7aaf15d0097a0229240c51ada6ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/287cdbafd030c8d331e4281b5aea1cc5e5467451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/6daefb5b1216ceef192c4385d98f60d247d5b6e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/7c1527b17d170ab4b59e65955b91bccfa80624e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/931303fc7664e1f513ed1748ea225b0c1fd32d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/9d4a21e122d78b4b0b314cf955d92d67494be94f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249569386.95/warc/af121527d6cef77c8069512cf87178d073f7efbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249578748.86/warc/47bf8c0edeb837d805be37da5b4db54027ffe0f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249578748.86/warc/4c86e74bca3c7fd033d134ff826ec1d1e5b6ce5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249578748.86/warc/7b0985529fa8f22e30e3eec039fdc5ff423aec65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249578748.86/warc/9b687cce62ac1ed430bdc9fefc62d078f5730b0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249578748.86/warc/b1f68b6caaef3470e5dea9a60f6661c2adba9de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/13798bee6fb7f90279ba740146d7b5b647b0d1c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/469bd943c1c56c003933ef1ba1dec2e0480688ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/4a5ecdb6037a771f0ba0b6968caf245f7333c64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/6839c1a4ee9fa088baf28f1a45829b7bedc5ee7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/73d84d9fbffc890dae9ac7c2060d5896acd0d87c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/1550249595829.93/warc/ebbbf5eb25ff0edaf3b13d7130dffc4b937f0930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/21e6c2cf976dd9585fd56e6a60c8a7f9b61bd5f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/243215deee9ecb1da4f6d016cfc37edb3d0541d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/24892b227adffe6e940c91e2bb1e07248f1df380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/25031316fce2bafec5051cde1437a7d3d46ea4e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/2545186f30d5cb2ad7aa63172343168baae4693a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/269e072941d97af8a6fe3cd5e966e4b8ade6e3b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/299dd2e9b14a4b64d2b905a428219b12a9fbe4c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/2cd5498f834bd2e229922c91ee81fb3ff633a47c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/2ff9e6ec816103be3208742cc96b74ead701e9ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/30f4b91b55a99676a040081c248d0646c054443b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/31c27c77a9a89473717534f2580346f4cf6f3316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/3578052c1c1314283acd4a79ee6f8f1658b39879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/3b189ea306f09a13e3221c41f755605f8faaf34a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/3c005359a2fc43bbddab15ef91d6a7c04e89677f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/3da2f41bd02411d338f54ca9f1d54fcc2ba8b33b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/3ee33600201c7befc5c052868697145567599a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/4a8adf1ec4ff71423b4fe646a79ead6773853ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/50b14bbe109ef4d2bf535d84c5830b1bb4b89aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/525442e76814d27507d30048800bf6a3e183510d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/52a36d4d01f8d194d3bd251c32d70cd38a404b7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/550a288e4d9efcf48102d3a5ff2ec05c9b213d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/57535f380e92f3b944d222f7ac221360859a48c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/59c694a938b4fbb7921d8c43bd63e346b65f0361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/5c153082ad44a9e8d8afd49c7a43d4a3ef83dc36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/71d97599c1b063a42d15542f42230ac620804a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/75ca437fec1831c5a329adf3df794d23fbef5708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/76f6d70a1df2ee40627f8553f902fa640721ea9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/7add3259a974c32a7b5a4d08b0810c95f84c4b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/7dbacae6b217993657bef2ebaef38bad9e0e82a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/85af77b93e1efbe3d770a8e36f2269007131c3e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/868c65d99dc37fde93685a779bf4cf9bfdb21434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/88d69d8287704900b667a6e6ac819fe10dab7de7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/8a00a33d4ed99976210d52727d4a27b9f2920743.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/8adb8f87f6dcae28260d9d7e966316debabf8de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/8fca6b7312ca0121aab2ba4ccbdaff0588f83dc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/9396286d697b89e4b1d443114371616bc58a7882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/948afbf3f62f314d6c7406a41bb6bf43745bb24d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/9759b8c76eb722b8cc32f6e4fa4d303e8690bba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/986ed3460d117f6406a939887f5db65df48ac44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/98fc663c7001e35f8a5977a051d29274246db6cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/a00cd4808bd61e71aa0dc4e21c8e2210f90db519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/a38d62aa212083061d8fd514d4fd47b2f0c2fda8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/a7ee7d27a2bf641c2bf91d4adb015bbf0e6df83e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/aa8cada15394355027d85421e2323b4773548a54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/abfd8859fdb2f011aaa2a0cb065d1800b7085624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/ac13772986a435ebe31ba52e775b65d9f30411df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/aea136306e6de5afa6cb1019c4bb5187b5111b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/b0a4047af8730e13c47d8a2162da942005b70d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/b3129b7ce5466270ab9d580340fffea5e2149f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/b31acd9a229adf323b9ac7cdc99b8db726c9773f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/ba1a680681fafb535d9368bd4f513a9d9bdeb572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/bdfd7879c2300031916a1958b8c8cdd6b36dc998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/bf210e7c12c74c2d35a44a11a0ac873109509738.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/c2dee3de62d824f9fe9af8edc1fc98ff008387db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/c8224418f8e436a2894a62a99cc5a119d09171b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/cd6278a9533d26fe56cebd468bef6b922724e640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/cd72a3800d38712253d305929d05b8e3b6a4c3b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d28126b0db625fad117027c8f9854c7ce75140b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d2ab3c6d03c3f76586c3a7d6a404012d5c58e340.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d44e343e619ead137e441c339d61ab127e50144d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d5364ca536d401d43dd42b5820ca694094c1785a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d6426dccc7b00f0398fd7f907cd2c58b164ddfc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/d79fd0661e780d704ac300a684942fbb39f8a6eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/dc0853a976570a78889219a980e113779def6a42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/dc506161eacdec805f6a7d0ed4bfb3b26f72f6c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/e249f21469ea8a8c053b9722d8e3450732932611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/e2f0fcecd6e5d3d8f8731e481ed6bc508d43cd0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/e81720b5a69f23b628bef00c5e3e7c91c6d06302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/eb0269857278157948715cc4aa3c2f9364d7a670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/eef942b32e564c00df52902c154622accb5ccdb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/f43284021864b6a8058356edc8771996bd96e5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/f7e2879efc71257865f2d2f4e8a505613cc29e32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/fcfa8b529452576761f0ad0adfdda53c653e1df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/ffef37b2d5279856c95ce10a90586587d4ea190d.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-09 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2019-09/mix/input" + output: "/tmp/cccc-CC-MAIN-2019-09/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2019-13.yaml b/configs/cccc/mixer/CC-MAIN-2019-13.yaml new file mode 100644 index 00000000..0851701c --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2019-13.yaml @@ -0,0 +1,674 @@ + +streams: + - name: cccc-CC-MAIN-2019-13 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/03eade6b92f8e8304ad2fb80ff514bc65e1a872f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/058e2a82174455cdde6cc3d79ee7c9b224bbb9f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/09f2ee4272d94bc396568b0ed6e081b8083cbc47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/111d2511108ee17eb2dc1c017c98e6b6826d0186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/11c2110828385923365fa37629b070c3d4e6a1f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/11c6344beaf23eb0e057898aaa7a0294e2a6342e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/14cf7f324ca0568f067cdbe7d38d7eddc7be8d0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/14dad08c739d0af5fb2eae190dc6799f71f9ffd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201329.40/warc/1452e0d3a3023bbc9560e30f1026f8a00ec4b3e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201329.40/warc/1582bf7db2c4b0b910daec2cff19888b183a3411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201329.40/warc/9abf66aeaddfc7662351a544a722277906630570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201329.40/warc/cfe590db56671b6e3d73355e17dbfb33393746c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201329.40/warc/d69ea8695c2295be50b5dafbda7db2c3038cd0f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201455.20/warc/352f7398f80766b59c709372de22d29c127dc9d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201455.20/warc/4afc3dc6bb81cafad8ff7ee5eee5113b0e50b6c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201455.20/warc/8b921d532668c36a137fee8e3ab28695f8438906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201455.20/warc/95fdce0ccfab6694955999ae81b2f62c3ed6f9d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201455.20/warc/980edbcfb7643d0fa80eb8a49cd3c8dc410a95ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201521.60/warc/1028cacbec87add321d031d9ec9e11bf14f9a264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201521.60/warc/a563baf73f14ee2f1f8c93acac7b64c4dfd049b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201521.60/warc/df51f2a8d9ecdc29f1a5a519cb064b1035a0cd34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201521.60/warc/fbaa155d7fbb87c524dd9780934d0ff1e49bc4af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201672.12/warc/4c760b95ca5b94bcb9d82dd1bdc5d0b75e3d716f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201672.12/warc/524f2d209a1508d9407ff524aa7d5304878ea483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201672.12/warc/ac2b99d2b3479b6234e3653e71e2a7b18275cbcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201672.12/warc/e999ae96e01e69f236188f93fc0a0232f0c90f4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201672.12/warc/f3e0c42f159bebf8a1ee589780eb0e1f001dc2b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201707.53/warc/047634ae9c578807b659fd27e1dfee92481a3347.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201707.53/warc/6201a4ea0cd3e91fd2b5e3001e931dc4ebc53967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201707.53/warc/7f53e4fee4fd0fde517be7d098860ec7ff3c0eb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201707.53/warc/c95fa05d3e09cff51cabe9f976c6273cddb475ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201707.53/warc/f933e3fcff79e15e27d394315809e3b6ebb28fa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201812.2/warc/27d76660ef5c3d14f85b09a59653cbb8ecc1bf15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201812.2/warc/2cf2dfc387fc69d6e43111fc2be1713099f0f04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201812.2/warc/7657e1daaead7f8af9c4c1b85e723375a3b37df0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201812.2/warc/c81db2d6b467a79222732ccdbbf5573d752f5f1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201812.2/warc/e6548053084a4470dd03bb5a7597d382b16fc158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201882.11/warc/4f1b744824092b8ea462eb5dba06e43a7e5bce5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201882.11/warc/711276bef716e32921b77fb897135c8513888e21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201882.11/warc/89754a866e92c506c99e6778c4df6a9b75463e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201882.11/warc/e21327a44c2afec6e1d20bc1ab47c4db8d3e886e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201882.11/warc/eae24ea973e4946d52b7c5677d1ade3f2e7fe8f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201885.28/warc/31c24d90bb5698faf1ed68ed187eb5afac8e5d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201885.28/warc/52c1cd9942250317dea33547d236f458e67d3358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201885.28/warc/cbf6973e30d0938f183b5f814be07e95e2d26ecb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201885.28/warc/fc4a644ebd5f3c3be0cff733d0d1ffb428dd3ff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201904.55/warc/074259e8beaa413a3d3f8cabecb8def18c91d1a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201904.55/warc/8adba02e156627895b00488a7cf6568f67322437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201904.55/warc/a992967f4458892bd4fc80f72da2d9295b4bcffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201904.55/warc/ab3d92d65e675dce61640655c9469a9904f765ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201904.55/warc/b24cd346511d06eb5c8efd07ccbbfc8be3979656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201922.85/warc/2af29593f52c20187b3ae02c82dfd762f4035fac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201922.85/warc/3dac60f7cfe74c18445b4bc084c2aebbe1b7ff21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201922.85/warc/68a57257c20801132d9e272d3113229d07ccbb4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201922.85/warc/96ccb08b7a03edcc8884db56ad45435e1f108fb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201922.85/warc/af056cd65919c786dd3653de7a833b3ffc290154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201953.19/warc/55e1283924f3fd1104aeafb79b0bc712b92dc418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201953.19/warc/5622ce6ba3020faa0b35ba27697db6f0017c2f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201953.19/warc/82e8f12462bb4ee25c33c76e507fd8172f5dd582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201953.19/warc/c98b82c45a9a10b7230524798e04e5e93853a440.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201953.19/warc/ccd7d7e2bb728bd6f5f804ce6ab7ee8f5f121c2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201996.61/warc/1059cb265ff264e4faf10584cd80def3323ee7a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201996.61/warc/90ded46b9adfd7bf0bc050b036a6048554a1fdc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201996.61/warc/e32444f1c54059f4fecbf42b6758c79fc059f3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201996.61/warc/e7a9c770520387f2e43753a718342ed54d5a1d4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912201996.61/warc/f6f574bdbaa644de96295cc1e6cffb9376863b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202003.56/warc/216fbbf8b99c7d4b28288cb0f0d8f4dcb4ceed8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202003.56/warc/25df1070db3b2bb8e02c9a1624c3b2be9fb0c618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202003.56/warc/3382d46a41cb7c031a5c9c65eca18559ea817b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202003.56/warc/bdf3f67f0cf739b117b67f91815da7de2aa15d88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202125.41/warc/0c3bb551b48a821de80e4f8947517a2e6e5ac712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202125.41/warc/14cf3bfc0ede3b0c57fb3977ce4eeddac62244af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202125.41/warc/440b4f3035d52435e8bb42129830ffb3ec770f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202125.41/warc/eaf05f652e51bec4ba5daf6b387b0ca90be00a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202125.41/warc/fc062b348245d2208ce731e9ebca12eb6a9a069c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202131.54/warc/223b316fd251ba38024469307039c223a12e7b6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202131.54/warc/88c3e372ae96ccb1afe21e97c9c42084069b38ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202131.54/warc/9e6dc8a990f41b6ce8ff93e67f0dab328146e951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202131.54/warc/c8baf5ca35f8aaddb5ba89e587730b2908945781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202131.54/warc/eb461ef2a460db323229ddc893f291a319b93b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202161.73/warc/37e3d780ea640c898be5290d12bb657424ae5977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202161.73/warc/58a8da96fefa55ea0025a1612b333f72ea9fadf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202161.73/warc/693d300243ae0ce1a8709a64b802492b69b31d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202161.73/warc/8fdfd7b0ac5d5245dbe8ebf96d18f899ce734be0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202161.73/warc/ae97ab180c89b3565a71954feb854b3033b6968a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202188.9/warc/09f3436fd4ad11bc9c1ec89af102abaad26ec7d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202188.9/warc/4aa42b87ac9b39143b4a3bcc5ddb093e5a938cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202188.9/warc/6ebc81c426dc54d86a82cfd8a5a6dd8eb0beefde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202188.9/warc/adfe12c1748d9e508397c9355f40209bdf658112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202188.9/warc/d15efa0a6dd79930aa9eb0c2b3d7b009ba921826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202199.51/warc/43a33a967f61eea36b45c623aebf91a5c0067588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202199.51/warc/4bf1f1f4fd1ae9dac5e691bada7d363092be8bb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202199.51/warc/8200dc014275cef99069c836cfc84b2ad4d97b14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202199.51/warc/f046ef16a105310c1c304e8a98403afdb21e7175.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202299.16/warc/6ce60ce74dcdf7f9eaea997c0502b3342216a6d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202299.16/warc/840be3afe67466d5a3fa48de0a8b520288790ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202299.16/warc/86c5455f0e336b2f31b445dc0bd94a531cdda4ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202299.16/warc/bafce44e62e2becd9ce99549591e832c4ac572ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202299.16/warc/c7fc386152e905b7b561e4f8bafd02d47836e938.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202303.66/warc/189a3da5fb2ee8765f60c1d59783f3209c2835a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202303.66/warc/6495ac78f43b6a0cc682b1bd2d5066d631f5567e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202303.66/warc/7bcdb72d698fae3733280f0294dd0c47d65dc21c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202303.66/warc/b65f8ee5304b69974348d736a81b6c2322294180.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202303.66/warc/f3ba4a079dd3adcc04513e3f08b3c2cb8792d0b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202324.5/warc/24a8831a439557ae8e20b53c3f79720f8d2f02bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202324.5/warc/34d7f76e4355a834a96e036bead14a94a6b98e8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202324.5/warc/95a007cf7dc632e4c7ea9d897724c383d05a1939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202324.5/warc/c37c9b16c002024d798c4fd08078024fc2a70c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202324.5/warc/df00b7cd549550cf0e04a7bef91c99bc2b2eff00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202326.46/warc/2e89243bf83ef476b711ae729b938154cb888d7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202326.46/warc/69f84f69834a5908cc1582f2a128a31e194cdc91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202326.46/warc/9c2c20b67ac160ccd81ca78b0d6fb31ae0e506fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202326.46/warc/b88e0965e64a55a48a0e5bbc87a550dadd5a6d27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202326.46/warc/d881d576e347b1caf1e28e797eb1665b82137682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202347.13/warc/193d6089c4a1b10372087aa87708345ae25a8c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202347.13/warc/709241a5947326104df55adf483fa7aacf5dd397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202347.13/warc/911377ff518d52b6b36162dcbc00373217622456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202347.13/warc/c2a2b353a4c527804a2c1b343335098701ab3e61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202433.77/warc/22054bdcf77661d2c15dcd66fdefc089224d50eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202433.77/warc/74702dac79e2826e31362811654a5c4801f16b47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202433.77/warc/a46c7dfbf0a8d8b62dccb7535613c5010d5c826d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202433.77/warc/a4883a8fec7f17c15590b5ffa6e5131ccb698c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202433.77/warc/c969865c9b2668533aa82e1985490b07832c3769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.64/warc/3b62914a8b302c04991ac2ae99ac8b481af315c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.64/warc/7354f9746ff77d3bdc750d3e76a39dee832082af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.64/warc/97039ff03c3f2de943f8cb72e39218f350e310ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.64/warc/9f450b5c20788a9043fa74521b453fe469147645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.64/warc/e1839ebc7c90242eab7b901c899eef6540157178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.86/warc/152dcdf480a399b2f053d4e17de0c012e6ac9fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.86/warc/3f74eaac39c454e4d424eb5df168961d16f82228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.86/warc/5b5b833794739963ad0735a9e9d1a0e34440ae2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.86/warc/a13644f55fb95634823d92812bb7142304208a9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202450.86/warc/a24e913256032fccaef282c3fd6797ef03f55ed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202471.4/warc/02433b7100d84a68c552ab07c82df3ef8bb80372.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202471.4/warc/5854b4f3f32a0f7d8090c25c117109c7c6a1b134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202471.4/warc/5f997916579f03e5621b46e522fc265b06b8e10b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202471.4/warc/8bea7e3475d8aea088dff4d0098c3fd1f8a5b787.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202471.4/warc/e6bbcb50f1c3e235d2b395232d7a763a3a7e6640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202474.26/warc/371b06e73abaae18b8bc2b770682f4bae943105a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202474.26/warc/3f926a559de2b4bcd3372793589153e0bd780dad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202474.26/warc/ab3993d044f6daaf0d6b59cc52d2ea72fbe2ef5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202474.26/warc/bc33a39f945d856b961aba4aa23a1f108e8f1d48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202476.48/warc/34fdfc6a4548906944fe2aaf12156151d8d166ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202476.48/warc/41b658e2a1517d098551701b35b3235e2051003b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202476.48/warc/6e541a6e2f144399e6af25499f2473b1a67b5202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202476.48/warc/c25f653e9b8b2e6c980490426cb8e7fd4f1e96c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202476.48/warc/cb3dd1007f24c25da7c6821efad6637e21a68209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202484.31/warc/68f473377c67bba1f1fc12697bc18ac794eea154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202484.31/warc/88f6b2effc4024445d7c1a83fbca50c47dff623a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202484.31/warc/a3a633891a50b5201eff1c5f8933274207729e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202484.31/warc/b1588fe2f55ebc719f82abdea43a6a5aa1794fa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202484.31/warc/f6eb0399479fa8f406a722117a63facb0c226e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202496.35/warc/150dae6abaa4d7032b6d9597cca4f8235657865c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202496.35/warc/4ffba58da6b4f0668d3ab9da9543e1e858be49e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202496.35/warc/deec89c5765e93130502a82f02668f58b1e5eb06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202496.35/warc/ec646dc76ad3103ca15ca53ea3cee49146413512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202496.35/warc/f551896524cccb7ef0eca40c9b61da03c2b89eb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202506.45/warc/64967b0ba7fa8804be2edd8837a6c84657a74b09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202506.45/warc/86e238173d4f096300e6539076ba373bf614d963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202506.45/warc/8762cfd826db3490fa46845f3a347bb0355af30d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202506.45/warc/87bdcf52a2ada7d8ce5d811b03ac346497190903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202506.45/warc/dadcb2aa9d068b8df92082a353a084d79c0b1db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202510.47/warc/001159b7fdd89a5090bd2d0e11171cbf6dd1e2bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202510.47/warc/0e87a561f781118242a9e38f0b790cbbd98c2e58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202510.47/warc/596cf6ef37c107b87b9260ded677c4b7a2ad2840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202510.47/warc/ea7b4e6e566dbe31d99355530f5100cc63f20d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202523.0/warc/0499702b07a30e8dfd488950ca2ee999267b103d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202523.0/warc/26e3e4666bfc050ed60268ff90a1599629a24a61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202523.0/warc/3f3aaa3cb6de419eb23891d44dd5591d1848b3d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202523.0/warc/cc2bfc821d4d6e84aaadb5e424ee4deee8625d65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202523.0/warc/fa200476e632ef6ca9dc60066a2d62d6dceb657b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202525.25/warc/14ee6e44f3ddce6a21280ce01a056262ceb7b26c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202525.25/warc/27159bf8b92eb7e3c3294edd94da58fcd63b94e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202525.25/warc/34144e18fd1b705815ac9db518ae97a60db47266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202525.25/warc/d1e2f5c2362577cce90002b7d73e2371b1b483e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202525.25/warc/d8724c5c0e28afaede2231802af6ac753610289d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202526.24/warc/045e16c12518d68426e1ce7520d3fb377a285b65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202526.24/warc/0f809e0213115015a1a1b42f23825462fb42516b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202526.24/warc/d506200efbeed461351f08a2d27760025f4e4332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202526.24/warc/f37336a1328e4857fe675c7220bb2e95d23f3dcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202526.24/warc/f5e44bb9108553742c56ca1ac6104c2bc35c9647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202530.49/warc/053d60ecbd4ebd6dcffb5d53c3bfad3e6527f7b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202530.49/warc/51a400709b130a52e8def8b1c702874591a443a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202530.49/warc/6eec04bfb31a06f2f4d1af27939b885db6091603.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202530.49/warc/743cd9ef14f9fcf5e9f7921d37b5a944eb6d74c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202530.49/warc/fda0e37881ad0b74b08dd8d73eb7a622360616d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.29/warc/55ca4236da6a92fb0b07f286da66e052d79fa980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.29/warc/562c99978c3500b933313fd290f019bc36dc86b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.29/warc/5ef637622425e1474801da8bb3d0f3fd55ec6aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.29/warc/a7e1385c14ccb3a512027d22927db5186a740fc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.7/warc/3f33c0d465cd0c1f9a839dd2fcf9c6f59457e29b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.7/warc/6d5d7955d716af45c58bea644ee128c2db15b5bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.7/warc/8b1d2a2d372dcd3233ec1478944ffe7f795899e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.7/warc/bfa61d599739aaa33a6df789bb34cc3b47bd808a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202572.7/warc/dfba2127464cf6e63a8eedd279802ee4b52ad722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202588.97/warc/0e0fe5f97e2c7b2ddca925e4bdadc722378d1496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202588.97/warc/3a3410635c28a4eb7ebd50f99d6642669913dc0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202588.97/warc/504df7e65aa28e6801f52eb91987809936cc3318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202588.97/warc/ca13b900b1eac89e1b7662835a1e59d0d57fd362.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202588.97/warc/fbcc939b8170acd56077c68150338527c40a687c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202589.68/warc/035911fcf7402d4665b254146163d717100603ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202589.68/warc/33b7a138a38f166e7587bae54323f63e8bab0a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202589.68/warc/646067bbbca3ef0d4c59813f0d1a08f281bccfbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202589.68/warc/6916a3e1a3b2bd0e55e355cb3fb11150e07fd300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202589.68/warc/cc225205a4875558e1f097986d4cab55cfe8c3f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202628.42/warc/05027df4ff5ae07421714039b69310cd41e79eab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202628.42/warc/40d3775555e2bbf3b93016c8c83d7d024297052e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202628.42/warc/9b1bc634de168ffcb8bb67a6d76b5593bfbde70c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202628.42/warc/c164a192729509268a436750eca4425e98e0a759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202628.42/warc/c1f8fe78a7f75c5e7c003e9e5328fd2acf5bf9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202635.43/warc/4a37c58e55a1f6be55210fd88f5c96696fd748fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202635.43/warc/7f2f5c447129bce09a1cfaf503e67e77154ecf35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202635.43/warc/98a968611be95baee07b2eec1e8371d6f1614ae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202635.43/warc/cde32ee92d92f2a98dd75b84c48fd04c8cd0a79a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202640.37/warc/b1dd5aa8b06f27d18d6a8f40f6f3aea65495171e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202640.37/warc/be85a480d57d31e436fa0af8540f231c087ee867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202640.37/warc/c5d317ef363e2770b8a38722374a3b5d4d7c6c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202640.37/warc/e5b5013d71baf94a33ca93b126484a8355ce003d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202640.37/warc/e72b691a16672fd8f0451b0df72a890470e94079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202642.32/warc/11cbb6679e94483539fee4d51831b29c0da0f813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202642.32/warc/42dd4f85aa58397e5922adb210c0c9f6c33af744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202642.32/warc/76ed8522157f54811a0eca5931dbd1d7ee8d0508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202642.32/warc/8629628b9c853760f1c5678bb5f670bb42b7dff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202642.32/warc/dc66f4cf5cecb99d3a6657f5b6f5bee562a64acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202658.65/warc/0b89891977d4ceebad574250b1be6a8da33b86ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202658.65/warc/95d8641d66c26c645eb0dd52745b0d755f61e062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202658.65/warc/c956009593fcbdd387732990b50e2fbde3866008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202658.65/warc/ed8f1e89dbbc2a9f08ef41f90e3549d20e740d7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202658.65/warc/f4ebd0e5e4d2b6d12c84b525465862e3c3d3377b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202671.79/warc/24598caf4fe4e449c552f4ff4bf8520de9bdb95f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202671.79/warc/3536ea8fcee1c43ba05f4d2b81f2ee87e3c39930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202671.79/warc/58fe9d61176aba1d11b7a34c62ec5d32f94c50b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202671.79/warc/b8f8c4b5fad552fe3111374578f9f0e3b781a27f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202671.79/warc/f49ab5794acc7e3ebca94bddf449254aa8635f13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202672.57/warc/47c3449863823bff6b72f0a1a99bfc7bebf680aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202672.57/warc/784f029b832998fdf2799f29bcc39803e5667dd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202672.57/warc/e22c1f54769561c4de91328bdcb94d93d2bd3da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202672.57/warc/fc0a14ee5270890a94d5074ab35f0a5a89aac4cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202688.89/warc/20b689dfd16bc5b7f1ca653f3c36d63d4fcca6c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202688.89/warc/26ac5d034737ff90fb687ed02873f1ecd2ef4c3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202688.89/warc/51da28bf7ed0444b4528fb27129c52e6b43fae0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202688.89/warc/64ad093901d991154d57b0f7941ad3acb2069ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202688.89/warc/882bb6f4b7b0e93308b2ceec5b90c6af7ff4b64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202689.76/warc/3adf85ff5844e00cded75309301fef7a2a3a3a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202689.76/warc/bd484e2f82a334545ea7d278e45a412d0dd73e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202689.76/warc/c7f4ebfb4edf2dfe84d5a7f5b70f8f1970af02d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202689.76/warc/eb265833e3c50b9267ba8dc086e693a74a1004d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202689.76/warc/fcd4fe4989e57d3ae7891d70f08f656806016469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202698.22/warc/66801f644c55af93b4ef828cd9403a5276a37b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202698.22/warc/8648f3ed050d929b9641117935dfdc0e80012fc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202698.22/warc/a3b363058f68540d1ff344b3e684a17ff9282b37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202698.22/warc/e651a50efcc3469c4dcf21114ee6c9c95d4a340b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202698.22/warc/f3219c2a5e5eab61d47e70181acf603ed767378c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202704.58/warc/1bcea3e4d573eb1c3a9b248ff4020e137b441272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202704.58/warc/37bcca3616aa2e6b70fa1c596e45d81a961a16ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202704.58/warc/9c502734f0df23c7ef52eb4553f9d51637530e65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202704.58/warc/cb3510c17f41f285b8920079c7293036a467997f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202704.58/warc/d045bb2c1540c72ab91436f5f2ae476d18b3bb71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202711.3/warc/137b6952383e95ffef5645da0367a1d494d1b4b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202711.3/warc/14a0e3d771021990d7cbfed0b7a45f341573578f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202711.3/warc/8e5cf820f3ad2117a8d3d8027f07de7869899285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202711.3/warc/d96550ac7d2c694af0a7f6c4795667c889e2da41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202723.74/warc/3d0c774d5c230c21f71a8260224ef891e5494a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202723.74/warc/a00ca06392437051aef643c1d8408cbe1524df71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202723.74/warc/a34465e1e8ab1aad9b2b3d807d9019dd59b0bb1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202723.74/warc/b3ca04fcaf3f82806c17feb436865f3a648aaeba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202723.74/warc/bcc95699d47a620d125242f5f50efdea8302f24f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202728.21/warc/4370dcf7ced8c68d6fa92e9d0cef02b946326c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202728.21/warc/a123d6d2f87f040afa74ee88170c30ffd5a5fd8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202728.21/warc/be66716ca4c5841079ca838a036d785a6505ad69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202728.21/warc/c5129a89f5b6a3dae2950c055be13a31e0bdb9c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202728.21/warc/d7ac5e087d129dc147c7864044d17d24f852ba51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.33/warc/055f4835dd3650ae7f8a213a3f4887290ac4afe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.33/warc/2dd4521786b68c30baa0c36c3bdd6ac995fc02a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.33/warc/5263a28fa302d533ba399062c2513c6c559562b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.33/warc/723c33900f13369d311b39c02d6749668b7bd81b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.33/warc/917f63ad9788ff69e667fc83b2ffc6ef55520d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.83/warc/35368b4beacf1cf7f199baeb2ea3ab9ab1c128ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.83/warc/5da196d4039ddfedf190c5dd6a186b6f4fe8d44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.83/warc/5e19dad8d969228ecfeb2e04e2926ada83b48e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.83/warc/e0808af3ff79630580e88d31cb4a24b62c582b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202781.83/warc/e12d81add27ffbe08961e3efa6f565eb8f8ac453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202804.80/warc/1491eeef36725c857f86ae144742f07fa6a12c10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202804.80/warc/1f35bac9eb8c02a19af05b57295c43ef5d72e58b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202804.80/warc/6e3833dd619871bb454886ebb634fbe9b15027f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202804.80/warc/f9393fa86f4bba33982d07c493b840f70cef3dc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202872.8/warc/1640c86504cc76b50f070689d832221b843c6811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202872.8/warc/48284dd573cfc08a11385384d4a0f1f16365821c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202872.8/warc/b0dbfb7b9aa74cf11d855e725db1a5d9a159f7b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202872.8/warc/dc3982bf34a7f12da5e9309d71d36eb0d74e0a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202872.8/warc/e0b5a813642230e1b090c8d3ac3cc4ec377788d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202889.30/warc/0758a911d1e29a139122d8d6cc2096b1ff615934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202889.30/warc/558f06d338a219c883fd85c33eb32f2cf3d2828f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202889.30/warc/636c767440a6c7d0f1fc87dc335b29dc836e5ce1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202889.30/warc/dc87db176d212bca2f9baa0513a38fab77680c2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202889.30/warc/e65d954e354934370a8fc1bd5d7334be695d5e5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202924.93/warc/5efd3b959fd692190723659c06dfd223c08fdf0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202924.93/warc/ce8157792097ed2652b92c98113e560c874c5a5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202924.93/warc/d97a4d780330cd51b154a946206f0f9ce703d013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202924.93/warc/e0e2472a9cbd88a4c7c7f3a7922f12f704fcf677.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912202924.93/warc/ec3926ac053d16ee7fa5750396d2dbe09c29e022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203021.14/warc/63169df8e82c54e32f4a7903e1e65e450b9fe643.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203021.14/warc/6e2689d19a8c6054bfab88f45da40daeef95b4e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203021.14/warc/8efbed64ce2b6b57c8b90835989e7b1decb55db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203021.14/warc/9ba9c13d8870b8cd7c06cdcfc10a37c42951f4d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203021.14/warc/9eaf4ac457ce98116ff4c0d2a8474c3703c7d7f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203093.63/warc/08c19742d53011a86529a5a9feb99fce21854ab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203093.63/warc/17329e43827d54528a987d702a5d91107c905872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203093.63/warc/7f97600160b691f3077b8c79b6da9559556920d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203093.63/warc/f1cdd41fd73e00969a78b36668b321bac04eb189.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203123.91/warc/671e149f399148e8ac4e6ab49280bd4331e6141d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203123.91/warc/67ca9a0fd7c342af793c194db1878d44dcc1ade2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203123.91/warc/70c79bd999956aa33482b8a4c993ac33033f2cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203123.91/warc/8706898162cb4283b10e0c826f89b811a75ceab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203123.91/warc/c8c1cedcd8fb639e0c5e1212ce229f9a68c52a10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203168.70/warc/5728023e403a9fe77877219d722c13460fb8285b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203168.70/warc/5cd538d2f192e31e596e2e2ae97a89b4536c7e80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203168.70/warc/713f71a8645f14c7cdf461c2860f301b6e89763e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203168.70/warc/d5efafc1cb91ffb426e470080e359b60d241cb4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203168.70/warc/e6bd479e4d6905a6d56870fd708dd223e90f690b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203326.34/warc/1de7e2139c5f0338fdddb8c3013e62fa70ee4da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203326.34/warc/48c926c8760c66d0e62ea5f0d12780d0195c536a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203326.34/warc/5d7c2e793dd98dd8fc28ebe46a30572672ca4aac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203326.34/warc/ae354a7f8fbbda6973a658931b8a3d20af47f10c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203326.34/warc/f81e96457058f6df8f6243b3ac1c7486f24cb57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203378.92/warc/299ed00aa51ed08e9df522627c6421b6aff1390e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203378.92/warc/7a0202a982ff3914fd28f9c4f00217116347d9f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203378.92/warc/9cd698c083380a08ecd04a9844c306b79f66a55f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203378.92/warc/aa1bc5ce14756d3ca37fae4235ddd10e47201f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203378.92/warc/bb5cbe5b1241112bea3f0d38f9d110dc5e4fa334.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203409.36/warc/9661be8b378eae66fa5d23ef8f56da7b47a7ecca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203409.36/warc/b3a560bbc7eda696f906703ee54cc5e85720c2a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203409.36/warc/d7d8b051b03918428ffa482162a610a2cd67fbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203409.36/warc/fa6f25635907d929af63adc26d8d3549eed02de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203438.69/warc/0a6ab1e4a1bec4e425868b811093ce2b870bfb30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203438.69/warc/1a531037773b521ce35226f7b372fddc063f2457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203438.69/warc/1c3e3eb493694faee0f09a7721520a9e45afae6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203438.69/warc/afef3449e9773d10d823fc42a5bd410016eedf42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203438.69/warc/fa68e777b8f85b267bfec533fbb60331a2b966d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203448.17/warc/3481b6c3658607d325465071ae939bcdeae71882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203448.17/warc/4dc17159c279ad1e0a9b92ca1e71d583b28d8596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203448.17/warc/7c05c41057e011f6825cdd9f23c07b851b5e6784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203448.17/warc/950fdc65a5b25fffdc2aa51f120b52e2a1045ec5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203448.17/warc/dc7da0dbc1f81653ada57e688a195118ee48e288.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203462.50/warc/3b752dbe280b1de85f56bc2e90ad0d2c0eb9a516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203462.50/warc/48d3e366267c20c8e3ad16271c85159563ea7113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203462.50/warc/af2013a7524398cbb4e9c931124df1af97d66fa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203462.50/warc/caf247c69245a1c3a4ad51fbe69b1517fc62df78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203462.50/warc/fefa3bf9bedd15665686442245a7e25bee2727e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203464.67/warc/0b39144a4de3240f4778dd30388e159276d498fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203464.67/warc/12662141ed09b78718c047ea471a8f752ff2e155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203464.67/warc/3fa21d4665566495c111bd9ce7e1aca6a3986337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203464.67/warc/40eb7d10c02f4cf276e94c4cb1e0a11e8a639f9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203464.67/warc/85af55347279b10f513ba99e7e95cc486f0a777e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203491.1/warc/2d313c471fe6a799d50f2453e93eb6ed770dc076.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203491.1/warc/3284eaaac42e2af09b5a55e23e63d73751739d26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203491.1/warc/7a28b0dbc4002f2b329bf35c9739a1ed9d54ee88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203491.1/warc/bb8506704404b5c9f4ffa86e56bf34616357f0ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203493.88/warc/415861a4b0923d5127802cce4c8266ba3871eba3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203493.88/warc/642a6d5a29593e450b4bebfd0dd3bf3bc47d9c2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203493.88/warc/b2a97e7d49d40faa0385f3bb13e12a0b20eb3f5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203493.88/warc/bc5bdf0ee563d3229f68e692b2c9621c16d3a574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203493.88/warc/d6c38d5a748a9ff93e75f38948e680dca98a327d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203529.38/warc/06b940a40d765dc9941d0d3e22c221451bbd53f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203529.38/warc/65abacc6dd921a55f5b63c16daf4f68ec272f1ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203529.38/warc/7e6c92cb1000c3133a974454c54526dfac760ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203529.38/warc/d5e7ecca8d53bad1cb4499925611882a2b8505a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203529.38/warc/e53a6aa6bec5e3934de9cafde943c0fdb9aadcd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203547.62/warc/322d0cd7eccdb3d599697610895bfe8555c7e706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203547.62/warc/4c84d85f743b5b350259e35a28ae82912b481274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203547.62/warc/be08351881b4f0c88303fa094b659cdd2000f565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203547.62/warc/bf681205b0a0652f8b0be9fd9308c608ceade800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203547.62/warc/e01bb2d321fb8b0a7a9b694c1df71ac6cfef7830.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203548.81/warc/26fa4eed75007910601ab387d0a43bd5891fa219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203548.81/warc/3303419e4bd5ef800d4a1d8aa8de3d662db5ee99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203548.81/warc/5b5273b7ee43b6e6d5fd8c2afe883dae69f8b83b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203548.81/warc/aefc9425b82a57c28bea9f6f90ad1e50410aa697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203548.81/warc/b3cd9a8b3cde865f75199dad278f7044e3c49dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203755.18/warc/27ec0708ad293ac51971646b3a8e03ce2cf1e41d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203755.18/warc/3cb4d1badeae82bdb09741dd8c62f01c7626af6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203755.18/warc/3d00856f0a3841945304508bbc0c992fa6af4fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203755.18/warc/8ef594d10723abf12ffa6fca3a0a22a9675d6758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203842.71/warc/2065736bdaabb05846ded55479655790251da2ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203842.71/warc/39cdbde4a86da5953a59c5e822e2f2412bb45a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203842.71/warc/67d99db83a46b36789a043432ea364a3307ba7e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203842.71/warc/8ff4cd258b42951bfeb464b3d50f8ecf28c06989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203842.71/warc/c749cebfc844348fe23a87c520d9d9d0d7e2aed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203865.15/warc/a834db30c7ea0409d4a2259c492b6951f9d70af2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203865.15/warc/b220a55909f05b7a13ccf3bbc594d33347f8d9af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203865.15/warc/c343bbfc8ae3d3af447f99c6d7403a63ceb022e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203865.15/warc/d472fd852432b961dbeb04644fd07d2f5c470007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203865.15/warc/ff5d59b5a689549250b3ee88da33f4a442cec24b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203947.59/warc/1d92808efe3fe1ab5b08f428f7e748bea014b3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203947.59/warc/2c3ed767eea401c5390d238628d2039ecd402351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203947.59/warc/309030c6d98421092d65abebaf19d54487fa9a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203947.59/warc/ca733540947195407f624df7f0957e3d66274da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203947.59/warc/f82f4703a3769ccb0a0b32bcf276e724883c5e3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203991.44/warc/0f25962aa627c1d1ee6f20314ea70197201efcac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203991.44/warc/3eca642bfae3905e0bf439b2d3bac5bf482abb15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203991.44/warc/b9138c6cd5b0f985233899e22b5d9c587fbc7c90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203991.44/warc/c7bcad4fc74126f4ab040aea671f45d170cb7b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912203991.44/warc/cfc30d7baba0cef316b567ade956d8014bda7c37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204077.10/warc/808eaa436f5762ef67a4bd92a609f975b4a58667.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204077.10/warc/cf5ded53b71e33af05eae55702b3ebeaf5c9dbc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204077.10/warc/dcf2905343dddf0dd991501da389da34d5198dc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204077.10/warc/edce75638a91d0c7a931d0ca261132aa30c2cae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204086.87/warc/1c9e699ea85a35ec662e025d5eae62b148ed2892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204086.87/warc/4764492c353c8ea7214d951f294b99ba58da5ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204086.87/warc/7dfa90c0e9776220cbc71f31be7dd680576e9e3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204086.87/warc/c6b8fe354b8aeee05e1bafdbaab8bd25fdaeff70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204086.87/warc/d63c5284b072f4497ee5263ed6924299cd3eb8c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204300.90/warc/4315c6e3f5d5275cfd43d1f8648ced92e3f6eb39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204300.90/warc/60897eb2aaea7af52480f37ea02ab5a3d4284435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204300.90/warc/67a2ca9eabc1db1f1c97dde81010b9d116159eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204300.90/warc/95f60e854cb4e77e32c95e4755f95a31015ab44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204300.90/warc/cb5cc9960fc360e11f0b23ce4672028df688df00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204461.23/warc/1a3bb97c3c139aa5e740c8b9cd26b5beb0ba9e66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204461.23/warc/89d5baca16a2fe155f6875414e267cd804045907.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204461.23/warc/8a2eb97e9a7694407fb5700d20db31c59eca5f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204461.23/warc/a3ff422ab80138e25cdd7c058e24ff126572539f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204461.23/warc/fc948d13128f0a724df80fe7d66610d7f906a7cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204736.6/warc/67fb75aade9ff75d7c1985e52016fd7aba4cb1eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204736.6/warc/9d45db007b5d66783c2e63f4a0c9c734c72b92a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204736.6/warc/f4028bce442d012200e1d6683c2fd8e1ca05493d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204736.6/warc/f876142a6c116c2228133ad8829b5e1ae83010dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204736.6/warc/fe201a6d108d2b56238567463785209315017e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204768.52/warc/6250695e5a048a3ed746b32b5f9170302d77b087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204768.52/warc/652dc347458ca47538dadcd1df8b88184c83af03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204768.52/warc/85fbaf93c5fb44972f7fdc8818adc26e2663d952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204768.52/warc/abeabb3ba8b48ea9bab98c18a37e391061a307d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204790.78/warc/21f23515010802bea8752132f878f442beac4fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204790.78/warc/455d88abddabbdba1e605f11568651250af07b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204790.78/warc/850fc8a526749babd0ec871c44e23e21cb188812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204790.78/warc/89e6ea29377162cb2e56861441b4ba9d55e8a571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204790.78/warc/db31ad2fecc0e96afb88bbb6245c90e3fe2cfeb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204857.82/warc/4902eb1b96694f22ca4e502a6c4f0cbcc4de94ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204857.82/warc/521e519ad5147afcaa6815877efb26f371bc4730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204857.82/warc/53e32e2ad5f45ccbc8dbd7aa108027fc66fc0831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204857.82/warc/8279c0e7a594f4dd840c324e5cbf044f2a063620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204857.82/warc/93be91d2843aa9d7fbdd11c64643f25a4fa70a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204885.27/warc/510326a1490026231309a60d6e704c32d122881a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204885.27/warc/624602f4b6015ad25587bad5106de7b6a40415f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204885.27/warc/7e338068e127cc9b1baba1397689d77696acff55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204885.27/warc/b8161603c08728e5ede30fcec2390534f7be9227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204885.27/warc/c61d1b04048e5a1007376ac6b658358f73328d8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204969.39/warc/12450daefbde5b8466cc67d96fb164959bea634b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204969.39/warc/1fb0d61d2b265e33100ca8487c166a1b8073ca73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204969.39/warc/916ed6809df6027ff967a8839401a2335149f009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204969.39/warc/a662dddfad29e7139150431b5214d58e0cbb0aad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912204969.39/warc/cc07bbe8acc183889a115d5106b701f1abef4bc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205163.72/warc/a72f2bd0aaafb306524684ffe786853dc209b957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205163.72/warc/b184fca2baad42ad18921e9b15aacb0437975c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205163.72/warc/f003331861f0e49cfa8511f410d17b8092282bcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205163.72/warc/f1a8d2dab988c419ddbe32dc3278f6ae02c59c67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205534.99/warc/5de53dec834443b23cffcf3e3c9dfe6e7778f3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205534.99/warc/61b28bd19490d2e5e874a7aa8f6ab11115aae05e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205534.99/warc/7adf94550dd8bb789e13df448e2e266192db181e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205534.99/warc/7d9a66a5a3b4b552ecdbdd041b67d5998099d062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205534.99/warc/a2c501ef5388b6445becbf9106be75a2c09833f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205597.84/warc/1dc0278244067803d6f10a2c4ea9dafe196e2a62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205597.84/warc/20cf421b32ee03052d70a36215de81b861c3935c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205597.84/warc/49440bb01a8bc30da80bc6c30f73c2c4340612c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205597.84/warc/7a2af33d1c72eabd1a2aedd7282d6e8d406c6032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205597.84/warc/9b87bfdcd1cd8f7f3bd87eade4d457a670f18e7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205600.75/warc/0bb6cbce5deaef0abf7202ee7098610ae0b0047b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205600.75/warc/10d28a47805bd30bfc235a2a90f47dfea4226bcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205600.75/warc/5a8e6d282aad58236bc2e356f9d1e6ac56e10542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205600.75/warc/6552d28908852f64b9d55a5853d4fe9593965680.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912205600.75/warc/921802f43c8067020aca9fd2176f6212bf8f41eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206016.98/warc/0750d92babdb948fe719d3eaa4cc19d3fe90b5cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206016.98/warc/739f263655a2d3f920a0fdfdc37a0fc7ec5d3f78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206016.98/warc/811a20de47ce5a61b60c8fb0f58281fc88d7c389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206016.98/warc/82ac6870b124d7cdd5fb321346c9cf8a2d0a915f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206016.98/warc/db5c615c7abf7ebd114ea4b8cbbd0761f343b1e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206677.94/warc/12d38fbfc4b33d4414ad2746aa8073309af3bd9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206677.94/warc/ea9f2f3b93c67fceb3e154c660308ca836926cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206677.94/warc/fb4b7993f823938576fcfb7c59a01981ba65e02e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912206677.94/warc/fea5a46d8892c9f8b87df8c7ce43bdb01a7b2109.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207146.96/warc/03506fe51b34b5a9ced88195794ff71d2b6ec719.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207146.96/warc/3f9be8ca37e6677fa88c3ce191d4392a6195e41f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207146.96/warc/4f8426e8a87238cc0c950d6df08dff7ade493305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207146.96/warc/a7382873c7a2be352669199861c379867522a73b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207146.96/warc/cc0deb04af00226de9f53559f7b4ff629ce10fb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207618.95/warc/100dbb0b415682956bc3a59d7eef0f016a80f609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207618.95/warc/5fba57b2f6858ca48ddf96d0ea49f34903b09d2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207618.95/warc/89edf3185e2012adad574c7a6ef36bc3a784a525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207618.95/warc/bb4755f381365c81deaf0f15d3edd87f8fad69e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1552912207618.95/warc/e0c11bbe7bfa5ea51eb45218bc372f3979c5ed92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/17fbe3d600deb7d8eca2e7df3bed4fcc1cf7c6f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/188e2c02da9c5c1b92b34fa71885798616a14cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/19556b874f0df88e7b2b3edd02090713292883d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1b9269e70105a42d5bb5404ebf2f4405f7db1a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1c1a8326579cf3cc52b6ac96c44f032760d091de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/1f777fccd70fdb322c463fe12dc3f5978d5dfdcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/2040f336305c3d07a816d6c0a897d65f35f93633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/264ffbf85fceeb0dc35d44433cf38dab66e9efb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/2736f82684d6c11fbd24a7b2b88f85e7c1bc46d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/29ed653b37d740490242ae16892ed775e9fc6888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/2cdd37e4c06a293873e5e420d972e55c662f8f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/3db8e6aa81636299f6f33baeea39b3865c6a149a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/422fbe391ad2b81cbc7e3833501d7e89b77063d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/435ac34567069b4e8c0a28f6013e17cd5d04c4fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/4548dc7091234baf8947b2523ef3cb5a6c3a4ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/46d4cd8adb21b830ebe910825ef90f9b265cf1be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/480b73347d9f28aa4c26435690d0cfe0198cd49d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/4a6e7cbb64af47f0cf5d9510fbe1f4921f65bee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/4b157b004cf297979d5fcb9085309d779278d0f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/4d5cf6d591cdce1ea3fdb496bfd1248c126ee387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/512dc838aec5ba49c5e62a8f06287f1c2709e9d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/545f8bf974c8b49a7b0752b4a91c5ae87a1e1d15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/5a2ccf8122000c7791b3ec24bd14230e115e0211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/5bf98a702b1b70f28a9eae0cbc51358cc308e44f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/5ce0b40b2aed66a702822cb72c68c13682f6cccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/5f7c5e5655be56ed833fbd9fd4dc681eca7df55c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/60d1881c98b6007b336aac287e2d886e7af5e4c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/6337b08e259b5cd02b60bdd3fc2f3cbda53f91f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/6346588c56d7c026137a34f70b32a7dc3912dcf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/64d07de9147e998c50799a2801722824c15c4e68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/6d5f7d99a6bd95585dc3fcbcca0093e463bf4d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/750dc0c8f07ed9bcd119fa339137faf859cce5bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/78f16cd47f74fc8fbb639e71e40b842094e4502e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/7ac106efb04d614adb66ae2b52f022dd7a8b3034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/7ac1a6bc762bf2ee5ab440a5ccb98226982aa087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/7bdc0c4453fd040c696011c5dfd2e8635d27c5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/7edfec22d0fe0a3454ef1db5c2963cd7f16c6e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/84a7477e74885d0cbbdaf8d476f8fe850dc76539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/870a1da2d9ec73e52bf152d53595dcc27c4067c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/8b8b5719475aad1cbe9a3031b4916ebccc87526e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/95d0c7c9ffd88183d3aba19efabf5da58292ac9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/98fc6c6dfe64770e434409fad329b89cd07db47e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/9ef1e952cd97f9098b1eaf6a19c6c0b37ce9ffd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/a19974971784ad000ce627a6e32836c27074f2b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/a27a1b93a5f0ffb8a4086cc7d860f191ba92ca4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/a3570a53b387b77389425de9e6dbfc45c7878d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/a3674260aee1697247e2b323bd187a2db818e582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/a36783812e64ee76c67f617c8d84a17b931f0033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/aa52715a83add1af64848526937f3183bc14643c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b1397937b9f61408d834e0b5a306bc8ccf82157e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b2104b80457731cb0aeb72978de6a88dfc045cd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b4fb63c91783a9e589788c17f6ddc0573d4968ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b595ae2ffaf942b4eff327df642c0bb9b5c120a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b5a309b9c8d499ea49aa071d4d29101f5cc77940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b68be944034ce359a92e3730cae4c9ba2abbdc67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/b739aa3fb7c9d9aaccd900e699b2fd296ec9baef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/ba449bd89547f4bc12fb5e55402dce4a3c494e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/bdb32c2097409e055284969bc624404c527f3dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/bfbdf3901ca7e8a6de09cf2cfb2ade688cf64775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/c612989721a37293e1de1d4c1a798a3bfaa972ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/ccd380c13897c4d8d5848fec960a10152dff4a85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/cdf987c1f03167d2d88a2daf08948b9f26aed03f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/cfa5e4876023576531007e422bc31d2083f4a077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/d012d793f235768d6accbfa0cedc29d7c75f42ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/d5daa68e10b814e708a66cd9277910ce6b7823bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/d6514ca7b1f6149fe26be499964d6c4bcac0eff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/d6e3e278c7ebeea8045a315c947277daf4ca672d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/d7a4d553727cfb2b20aed09643cbd3fc923fe577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/e749518c13aef1fd5a73435e5044d31a4defb065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/ec39fd8c898a28a90890b0b25e0359e864f9ae8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/ee6ac57155b296f8f146c22b4f9a197131011903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/eecf2c636669290e69f6c2495595e8d1071a7432.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-13 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2019-13/mix/input" + output: "/tmp/cccc-CC-MAIN-2019-13/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2019-30.yaml b/configs/cccc/mixer/CC-MAIN-2019-30.yaml new file mode 100644 index 00000000..ac8515e5 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2019-30.yaml @@ -0,0 +1,669 @@ + +streams: + - name: cccc-CC-MAIN-2019-30 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/01343ff42761fb71a6e89cd035e450a61927e368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/027e109de94118995d148bae2abc2625f030f1eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/03fb8b9cfc21f1027f17505cd7bd37a5c8438721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/09fbeef083a9c73c6709b3f343eec14e42495c14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0b13e924c1ffa2afc1a058c00f96f04c3a0d1360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0bd154107fc93146fc42f4923f0f0850f341bcff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0d01ddeb36f5399b7c8fa3a8f19c9357cb1971df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0d40f60cdbaf6442d76b4228f799c84a763b33c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0d7ef062dd9735fa9dec08f9021fcc06eb81fc03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/0f7fd7a905f79e28ba7db49c93916f2a5c6df03f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/122b39e16eef4d24fd17c7de203dffee4391b149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1531cf6fc7c060022877331fea075d018345a2fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195523840.34/warc/18ec14348ab0674189b3fe8cbdab5a2998aebff7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195523840.34/warc/3e8c28278da33a5e7c0ee7df70210e72f483df25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195523840.34/warc/8cd2651b17c0795cd51897244b17b49de117abdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195523840.34/warc/d22fe5832e1bddd582893fb5aed83a3e553ece44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195523840.34/warc/d3eb1e60d43a388b65239d0b489b5d7ba970c29c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524111.50/warc/460d1a7e0921818508889b0f880c9bde288e81e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524111.50/warc/48a11d323ca93067485c6b00253ca5011d49f563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524111.50/warc/98406ec37074337232a3fa4260781cc47759f32f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524111.50/warc/a862927dd945dc7bb7c0356fe0e6c51cfa77eec5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524111.50/warc/c1dd9d03b8b7b0fc2eaac8b0f25635068ffd0849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524254.28/warc/36d41cf8db15e5876b10142334e8a0f2d80cd5dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524254.28/warc/41d9bcc7a960c9ae622ddc4f06b8f15c9299187a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524254.28/warc/693f4c588b005fba4304a89724f15112c5862757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524254.28/warc/85db862826d0d883718a7d9b35d58f38f476c463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524290.60/warc/804d7e2c94108f8a65dbad9ce0842a9a31057f8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524290.60/warc/9bc24a432a047a48e2fbc84ba524eb8cf34f4690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524290.60/warc/c513cd237c7d77694ecfc8cdd91bec1d4d006761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524290.60/warc/c69b2202bdfd54f40f6a4c3ea77fcb0325ae141c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524290.60/warc/d367af73512acfc1f1016961f580071f5631fb4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524475.48/warc/1d42c2ca790ab911e17f33cf5a0dae9734b9eb91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524475.48/warc/79171abd39c810fabd7816281f449126f2b6cfd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524475.48/warc/868587b91749d40a133a9e1c81bfbd1410f820bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524475.48/warc/8f41281ab52be2eadc8b846c032abf1162617e15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524475.48/warc/9b1ed9ca09b8015239657ad84a3e21a705107a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524502.23/warc/1dd693b6d46e8a6bbcd7f064d65b9ba1961e2c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524502.23/warc/5514cbfb187c49bb56f38fbbe14f73da593e35ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524502.23/warc/89e95fb10ad934b480d3e9a31a543a9624ec29e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524502.23/warc/97aea22d5ffe2f1b39a0d69ed3e93751a6764565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524502.23/warc/d89e2cdd89707fd34a5c28f2f0836252a5ed7dbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524503.7/warc/1623170b1ae9f2fa196f6ea001f27e3589203ab7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524503.7/warc/6c67e8be50705d5a95214d07c3128d8c5649d517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524503.7/warc/9d807596679bb1fc4aa70616120a94d8383e6231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524503.7/warc/c4ff69a0133afbbe82c44ef2b01bc1da9ce1d85f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524503.7/warc/caf4f918217891a6b9e4a828878650e92900fe6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524517.31/warc/71e6226b011abbb50af106d08fef043759b8078c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524517.31/warc/c0cb9b95ef1181353d992a6bf9f6b7ffc7202ff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524517.31/warc/c36776af888b1e10a9a05fe09c1fad17bf931a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524517.31/warc/ebe26ba2e55b7627c27d0efc1772091a4ccc710c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524522.18/warc/002dee05c706e24c6f2027596c3bad69389ce119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524522.18/warc/1bb91ace9807f5c5bd6904f1dc51edc9ab3e0b8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524522.18/warc/62615b0f15c2859c166810a8f19eab025bacc136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524522.18/warc/76dcb050ad12acc56afe0c51d7c5aec2c840cc81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524522.18/warc/dc54cc445e414c9bb213630a81cab08ff76f9c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524548.22/warc/04125ecf466ed3dbf1f4025a60af006ea71ea9fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524548.22/warc/40c31456f15a9c0bac9905eb0cd74e030bd2e4ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524548.22/warc/77b25c9ddd97bf6c815db252af49cc576f42a0b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524548.22/warc/9e4a3a46257397736f47387aea6fd045ca7cd3e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524548.22/warc/fe8a55b04ea68f587cde1789bd7df6daf2629fc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524568.14/warc/09d8ab843cd3b201340435bc4ce5c2959b4a09d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524568.14/warc/0ce0a763822879194bf313544906f615553cbacc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524568.14/warc/21c4c0fc39c1be5ffc05967e094c611fd786763b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524568.14/warc/a1ded04967de8a7cfa299b8a8c5caa195109d636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524568.14/warc/ec8c1845ed03fe3be703556adbfa7eab0ce1ee35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524679.39/warc/13aa5200efb8da602b5a9599a9a6f223b8b186a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524679.39/warc/3e539de4add1cb6e3c9a2c50029987027a3a1a74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524679.39/warc/3f3f3935916c69f8f86074e24f9475f09248f47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524679.39/warc/4b93bdfe0de1021a5bf1ace845f1853065caf02d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524679.39/warc/f5fcd7471c9fb39f78f57cd985a099f343f69679.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524685.42/warc/56f45dc5f5470fcede149cb0b40b34c4edd6e9fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524685.42/warc/97403dd544860871b776868cb86b3b15ecbd4082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524685.42/warc/99607754810711969b15ba7c443d35e2d791537f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524685.42/warc/b3f06c5c1c94c7e08fa969f01a7386f8cace0813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524879.8/warc/330f29e6b517473b159b6e488dbf0ceb7b9c9cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524879.8/warc/4666f6f4f57c42c3555572198118f37a3256f0e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524879.8/warc/8716b51614867bd372042882f952c8a91a17ff04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524879.8/warc/b3485440a6d6de642de1f913c30914579671422c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524879.8/warc/fcf67fc52f992b4fc51a2909adf0c7dc36996922.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524972.66/warc/0852533d93fb6101d2b024308d35430e05fa091b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524972.66/warc/3d9033f97edd31d5fbf923823a87dc3f88ad57e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524972.66/warc/7cf9c641d79777e8af00a68ffab89bd38cbd6835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524972.66/warc/9f202a46936a8708f1a0b1733ccdadc6aac1f927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195524972.66/warc/dc7247cd387894a3b43373c758fb812ca701f705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525004.24/warc/3b94b47d2ec31a8f2ba522702a9f5141e46ccff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525004.24/warc/3f8f6e46bc84a7fabcfcb346eff2e0a03307f656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525004.24/warc/56685a7775fb0a7338a6c739538a04e29cf4ceea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525004.24/warc/7be1eb0f7d6f53ff50b55f3f460a2e75145ac4b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525004.24/warc/e8490f7b1eaaf19063f48e9b2ebff65658239eab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525009.36/warc/120afe7bc9e01b3883a0675c0a5789704495e748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525009.36/warc/346f3ac5f7fc3aee30b8428cc8d4518db6c5a85f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525009.36/warc/799e0fc3096334c5ac28efa93e8a2dceebc6ee0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525009.36/warc/c2f677978595a3e683cac9bebf6dab16c57c4652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525009.36/warc/f96e18162bd76a400f939f0eae8d0823bcdd9d5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525046.5/warc/09b51857a8a40280690aa44deae140189a6f79a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525046.5/warc/23ab004ea39f8e6a8bcda9362c4b6b297d47f19b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525046.5/warc/30658a5e5d5e4cb050bf818bf003e5050388b20b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525046.5/warc/e04a23207b1a45bd4009c23e125af30814611ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525094.53/warc/1a31727d64c8e76cd98900d72e64d834e5cf031d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525094.53/warc/1afe5aa83919596b34cb22e51d32f81efbad2e10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525094.53/warc/3ddab9bf8c98cbcb9558c29afec4a74de3b0f596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525094.53/warc/8c6ac684cd137aea4c6c97ee515a389319f63c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525094.53/warc/a52c940a9f7f3bae2ad699240fc1e7f9ad4a834d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525133.20/warc/25f062ad9403602e75dbfceb1e2cf334f7bb3448.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525133.20/warc/5963b73035e95b38e9d511061f2c49f51bdee21f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525133.20/warc/5ae1c0218cacb8258af6b8484a9814b6d28f9895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525133.20/warc/ec27f62e97c79b60cd772bea1cd14903d618b87a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525133.20/warc/f152b639fbdba77b7554b5b9854aae0fac01f2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525136.58/warc/28d3da943ccd72ba7e756a60ced473c84f151e5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525136.58/warc/57b6361df7c6339f21ffefa47cb036b684f49679.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525136.58/warc/61ce7f8a175092b451001fb749c0415570ea5b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525136.58/warc/bd30b4d87446621f76d006b4a5571faad05f972b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525136.58/warc/f5c06c37aec8fab5b58af1fd5928fd32e824451a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525187.9/warc/6a716ed80b6f7db6bf39eacf2fbb3e9abd3d789e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525187.9/warc/6f24ae35287df5977bfa685d4069591b77927c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525187.9/warc/7c795f90181c9db8a31bd13d44ee4c4e98766532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525187.9/warc/c03b2cea570a1e9965b0e4a8f950d473721399b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525187.9/warc/ea7a2cc2d8ae3dced8c31cda901d9dd0555bb24e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525312.3/warc/32efd01810757c015eecb0131f15d96ddb5c0318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525312.3/warc/b6bcfe731baf1ee15a0f5066494f6f70a5da6fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525312.3/warc/c354039be88922f671017bc56e94949dbb8bc14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525312.3/warc/eb7cf32fb2791a221055a79429726849af9bdf39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525355.54/warc/2450c564d475a5ec43e4fde73b3072c697b7fad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525355.54/warc/2fe3ecea11d16e0d3863e67614323acb14dbb821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525355.54/warc/7d3a8a43fc6ea30011fd42bbf6ece00a636caf01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525355.54/warc/bf7f91999a60736d02b6e83406540cd72a06c4f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525355.54/warc/f6d59fca6f01c8cf74855ae8d3b777f3daa22d59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525374.43/warc/2922f85118b1e72dce092c72f9b13a84c4a83daa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525374.43/warc/82896904d449977542ef31bf410f8c5cdd8cc680.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525374.43/warc/878d8739961c2e99c6252aa629332da1ce6a5366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525374.43/warc/9aea040dd5432b40f6c817bb02faf4a18798f472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525374.43/warc/ed9a7dbb2aa94bebc578a5ac1dcb1b6ea9b4919e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525402.30/warc/0b55ea83e0abc6fb860345f8cdfa92bd92914ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525402.30/warc/0f83b9003ea1d91825a50bdfebfe61ff28dc7aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525402.30/warc/703d103fdc2133d9d974315e1316d6d97ca6ff56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525402.30/warc/756c57e74d41c937b14b908c3db9e6279d8a3ceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525402.30/warc/c7641ede7762b6fe33f09978578ea205335bf366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525414.52/warc/0e35e385eb3a08067e3994a9ee33e307eb161072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525414.52/warc/8aa8154836bc9aa100cebed91005c63981e85062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525414.52/warc/bdf4ed4187d6f107f8294fdf16d96a18fe3ededa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525414.52/warc/c3f7225af1f18165592ee5cca25988fa6dac1b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525414.52/warc/f35479f043d203d19243d75e2b9eee110ad1e596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.62/warc/2067f9bb38bdf091d901ce62c34ac3272070e99c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.62/warc/9a08aeaeda8155e8e8b9bbf08eab6cd22487c49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.62/warc/a1f5720c012b2303d92178a6dffe98e44d0e3a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.62/warc/cbe567ec54e1d8fcc77f00b99186b4af709cc86c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.64/warc/750af18d4f68ee6757ff508db2008f94078cc222.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.64/warc/c6329fb01a810480d050f8f4cf75d431f5902a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.64/warc/cfda8aa624389f4f8aebbc8adbfb741d95c3c2ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.64/warc/debcb7e4246feb217f8d72b77deaae345f2eef77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525483.64/warc/e2641bffb4ee47da9359a96825d4e73c235a0c75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525500.21/warc/0941fe6cd8f7554f4b1df5fdfc9ef14186db476d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525500.21/warc/14018471b2e49b6d1fdc305ee2ec62259c25ee5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525500.21/warc/303d43f97890b07dfb4f5b1bb855c1e484702ed3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525500.21/warc/a43dc2163e9d67d292c392edb7cf964820e1ef4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525500.21/warc/c771f93b989c4f0a188a60d8aeb6ea1e7c8e0a3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525524.12/warc/73dddc59b04dc3db99b295be41960ee87e289a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525524.12/warc/97b879db0d769b654abc4dc8fc91b13aa7125e58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525524.12/warc/b210193cc24a881e0d14402ef06963d7794b9d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525524.12/warc/e31ef42aba339fb4ac2c39674f939acd8f3871b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525524.12/warc/ed439aa4838c9373689a4367c58c7eff4cb48af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525587.2/warc/106d58832afcfdd234503284d7f98adb40020c85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525587.2/warc/2c32ce6059750fcc7c3e2abe36abb44482e9f28e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525587.2/warc/305ed3483938d3e9b9633218a8afe8b4f5e4e0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525587.2/warc/4ec25910a267af8d2b5327ef22c381068e703fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525587.2/warc/e06fe51359f6a2d9f66a09aa56f0502fa9022948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525627.38/warc/1b86cf987888bf8cb9095692634524860df75055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525627.38/warc/2c63636ba9e9fb6f35f938c95948e728f18e40a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525627.38/warc/429e7537ed5c6eaf26696a00bff0a228b8af018e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525627.38/warc/476cc4971e5dffd9eebee2abf4761327a6980c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525634.13/warc/030e0c9344963ec1d0fb89aec3ed47b33ed2db8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525634.13/warc/3e8011c95117ef86ca1def4455704d2497401e77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525634.13/warc/6894936eea3929ea146648e8f7140d5a0d3d2e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525634.13/warc/d0881819e79a5503a28836120b21af157dfbb300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525634.13/warc/e2b28a5824482a93c8fb931dfea9e16a23a761a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525659.27/warc/1c79a3039d91b9c417913aae200eb02a513e3faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525659.27/warc/7167c63e25445b0c62da758ac08cceeb5c3dc097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525659.27/warc/73c2c0980893ea0cc637fe7cd2e22bdc4200ada3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525659.27/warc/9e79ada5c7e04f49c9030dfd3a36350c8d0dbf6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525659.27/warc/ac42152304fca3fbbe7b7a77dea8bef7e42198ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525699.51/warc/00d6a2aee83386d1f053a5cd57b3dcfa8d01fa0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525699.51/warc/541bc0c8d568727b5f278463dda0d14582dca630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525699.51/warc/7abb91241ca5b56585900beeef0531fe7313a984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525699.51/warc/a4aae3c058290a02e0a894d07ed932e248eb12ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525699.51/warc/c5a67a9f274323a6ebdb0958b9b1f1602df43098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525793.19/warc/34e1fc2c10779626e7c35980529dc8c19731acaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525793.19/warc/512f0be2f348c07577ec016761bec875fa953178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525793.19/warc/b4d56fe26b318dc7aa2a612cd6ed5af44c2903c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525793.19/warc/d5d42f2c45e146cd22dbe396b3f58e35c0271aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525793.19/warc/d85989d72351b0b69c16a289d6989b38bf4ff636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525829.33/warc/636ecdfe7fb343540c1126b3fb1977fccf104c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525829.33/warc/6b27f37c42ba4571e277cb6bca86610d47a30bb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525829.33/warc/aca67fb231c3385317a5033f6060ae985353aaa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525829.33/warc/ec36903a8e60820eec6799fbffcc61d5626b798c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525863.49/warc/1e479f47a692feb9921aa8fd70ba9df8c9df9d5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525863.49/warc/489fedae0fcd2fd1f7b9d4acfd4a2e1c01cd6cce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525863.49/warc/49b64790e399e892e743523f516764d303ff3d11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525863.49/warc/98d7a5d7550f06625f7143163ace7e7522b6fab5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525863.49/warc/b984602fac69f8ebe6c30681649aa3c2e530e27e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525973.56/warc/197e05fe2c2c5c5e8ae23d6d2c53ef450d7c609c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525973.56/warc/2758a63ff6c8ae16790594ad7e1cd31f118de345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525973.56/warc/8f69d9ac391f9c6a781b185ccf59c75f2a834e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525973.56/warc/c703312ffa979be48563740ac4176bb7dff78ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525973.56/warc/e12c504f2f71f9f647f3219ee9be79d0f8139040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525974.74/warc/ad41c0a0945c89e2561b89da06192982c18ccea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525974.74/warc/b5b42a383a9d589a6d8f068df52bab2a14d18736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525974.74/warc/bb7534fd9c3ea1e92a1744a48f76dd6908ba0518.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525974.74/warc/cee97e15f07a8d14aae506ad014696b3f4d28428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195525974.74/warc/d37a9f530a87e1e2313a300c668de52ad8fc1107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526064.11/warc/00d52525a23924361df1a7904a92887b276c5f61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526064.11/warc/5309ac208f9dffcbf239a61703ed0e3f00955b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526064.11/warc/80a82a4138ebc9714d52524a6a00203edb441e95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526064.11/warc/9aabd453b9c16e4b0d992fd578fe639a858e1c32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526064.11/warc/ccb8d59a6664e41ad74f6a4ae89efb4982c0d8f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526153.35/warc/19c7b79b6323cd9b1e3738e531ae323f114ed8e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526153.35/warc/2c71726ae700c2ae5a1940078fd1ee760085537e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526153.35/warc/3c5611e573f845ce5ab84f86942fe381d5fefc11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526153.35/warc/b572c09351143350414cd233f41b5e57308e4489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526210.32/warc/3475e402f786f52dd9cea27ea570608a0e91333f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526210.32/warc/90d0a0b242c4cca0342bc20f77caca4b53fa4669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526210.32/warc/ba99662cdd21952ebe42ddf1acf7382c3c4586ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526210.32/warc/e286668386371bddb9b376685096d35df006e74b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526210.32/warc/e3b13b747ceb0a9e0de82b428bb3a25da6218aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526237.47/warc/51fa5f50ce35e83e4be77e5a7a31279e3703b4a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526237.47/warc/b5f261c1a1fbf60dc3b5b800d347fe9344db186d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526237.47/warc/eb6fc748008d2884c5e8a4698436aafe938247c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526237.47/warc/f8ad679cbf6523a19f34e55f46aeb9d53f4dbdc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526237.47/warc/f916b648ca50fba60dc8b5169040f74979564ceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526254.26/warc/23889f926614acffb27625827dbed99771d34811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526254.26/warc/28a76154790f9fcdd3f1eb2568b0a8623f78138d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526254.26/warc/6f2454dc2fadd9ae650869364d05566fed9f66ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526254.26/warc/99cba2a96c68e21872b96594b29e7e770a4ca47a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526254.26/warc/9e6b8667e24cb2949a806ebfb050623d395bd7bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526324.57/warc/119ceef143da6760df1e975cb318af5d84282ef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526324.57/warc/7dbee3a8cc7074abcb9b34cb62f296a4c614f75d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526324.57/warc/8ec059ab36450cd86b10a2c3a1d172f2891b1026.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526324.57/warc/8f4a13cf4580df80067efa8def424ca0d8a76209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526324.57/warc/a9f86079e6a67d55410c7122c7e1d7ae098198dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526337.45/warc/597e6a0a80298c0069aae40bb4bcd122a2bc8f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526337.45/warc/94b20fc6a3b0e22f8d4fae1c489af995ee6b0a2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526337.45/warc/9da101bc5ff8119112bebd21ce453283336286f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526337.45/warc/bb5522ea062faf92cd83b31b0dd76d413f7d58a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526359.16/warc/0023bb32b7b0d86c1b285901054bef6ee6835cc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526359.16/warc/35314b3a59148e169f207952bef0d8a1b24da90e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526359.16/warc/38d61fb83f148404eee0f466f9ef3b7c72400c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526359.16/warc/3cc45027f6e069822355bf19777b533e12f54a23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526359.16/warc/8458bd31534b54aef56247dc468cd25a976e4a82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526386.37/warc/0c2ade094cddeda87adc30ee91fe862968e148d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526386.37/warc/1b4dbce2e60eee57ed380335b621f014c808ef46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526386.37/warc/444e2f50c0cd2dc3f0d074b481f9068d8f1208e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526386.37/warc/7314717aced719416057e4e276ea64c317a5d7c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526386.37/warc/e286389e8d3910ecb2f9e3bf9547dc45a32a6ad8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526401.41/warc/2703fdba702ef27cf1aad6841889823f047527e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526401.41/warc/33b28e4384a4728a24fa4cf73a8d53e18d219490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526401.41/warc/4d9106d92b60f5340e49416d711eab14a328cf96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526401.41/warc/7f3efbd622b3aed9fa2c1ec693772928513e4387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526401.41/warc/c6951c7804a34e557e21c55f0687c5fd9b5abb1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526408.59/warc/08e5ff22e2281161ccf601446423a74f0a57d1eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526408.59/warc/11912bbdec4b1487e6bf328f47278f93825fecf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526408.59/warc/15b9339f5f8556989515c274ce2408df3e6863dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526408.59/warc/af920710c55f67839e908eb91454a3db1f95055b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526408.59/warc/f8b483421401f011895cf538fa62c29d3cf68893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526446.61/warc/0a623eaf0cd49bb86f16fcc6d6650909aa08a33e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526446.61/warc/4b857e4ccded15d5771b8aac67c66e4f11272607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526446.61/warc/c5446dc514a9c4290430e6c33223435ce1268206.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526446.61/warc/c5f0dd7fad04435d46dc8bbca525b37916ab0350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526489.6/warc/04f7add3a7ca9d557d352cf0129d7b3ee0b9848f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526489.6/warc/0f98e2562f5d6a568016b9d71806e981271bf8d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526489.6/warc/a6ff6ee56ffff26f0bb799df0ad6c7939bb45d08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526489.6/warc/c4e8780d016a5018e7f77af812a47cfcb783e656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526489.6/warc/e214578d9339b915ded03c2db2d75666f02aab85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526506.44/warc/21d5ece9c122ad59d039469625afcaf686c97a3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526506.44/warc/468275310fa9db84af6b5c137f02c57ffdeac2f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526506.44/warc/85909f8da4521325985f84311211da543f4b4b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526506.44/warc/a8133a9cb68531c774ac628b9937d065d5448a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526506.44/warc/c9396fbefab23430c9183ebb17a4a803f8e90ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526508.29/warc/06da99be9adb050cd4731ef6f96ac435ccc28bcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526508.29/warc/371ac3bc681b3feef64663ee4441ef759e3fd39a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526508.29/warc/48fd1fedfc152fe3174cba0096782025673a1fc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526508.29/warc/7a46a953c4bf04f5d601d5c23eef6a2dbd4e1c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526508.29/warc/b4e13cb38a60d4ad09c6292a88f19f7f17ff0642.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526517.67/warc/5c1c20155c8cc7fdf0eb62d8a8f16f34f5d9538b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526517.67/warc/63e403856a32c70e8c62b6e2f8f6d27aa14c0714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526517.67/warc/7de2eed65558f1fb003b2732a067774e71580ce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526517.67/warc/a6dd67b11a9b9fa03213f0fd7f84367fe25a73ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526517.67/warc/adf55b34b200f574c82a28b13043d6324eb6bf7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526536.46/warc/4345b3e67035bb7a0ce784d8c205b449b9ad6b4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526536.46/warc/96b6a8e52e6dd40c429f5817094d765bd3d39f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526536.46/warc/e267b8f802171c69d98ca6b204008fec6637bf6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526536.46/warc/ed0a0296753f436257b72c27351b7517d1dc0bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526560.40/warc/1be792b2362f0e9d2b427ebc6cb44f393db19df9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526560.40/warc/5bf2fb3efdf2bafeb4e275dab69a950fb777f4f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526560.40/warc/a988dda9e82a402ad183a059dd47c9e54367098f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526560.40/warc/e9f276bf0e2445fa50f6f89138fb5ac12954b26b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526560.40/warc/edc67559fa9fdcaa1583e4c4b289e53879d9ba4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526670.1/warc/09a9909aff9035195ee167c1377b92e1ed090795.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526670.1/warc/73e85757681ac8b9c28acc3a3bd4202e42ef697d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526670.1/warc/b5869c758fc9592491532fc7e4781c19b7d22f03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526670.1/warc/f877ddfdec59f2f32383845efc6cd25f6503f2ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526670.1/warc/fb707d3777c137030ce0c71d6da3ae46dca6c919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526714.15/warc/1601782102e2ea5d9c108c4456b10b9a447b8389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526714.15/warc/668f477d208e83a1f8de17afab37f5b35c93e4dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526714.15/warc/7ceff9e3a03d7f1840a13f40198b591190a0001f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526714.15/warc/9a36738030d40f4968b442e9b2bfbe76a4c51cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526714.15/warc/eaae60ccd0797399da8b476a6b201fe046e75c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526799.4/warc/08b67bd921e4692956dc735f4ed50a29bd747ac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526799.4/warc/9b2fee22ca900ae6cc413d306b551382dcdbc4b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526799.4/warc/ad519a2e88813864223929b99411872bc46414ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526799.4/warc/b3e1fe3a08a064f7d58015cdd4ad20a9598ca919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526799.4/warc/d88c67872bd9d58e406f24e82429e0f1e5db235c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526818.17/warc/3fa32f9887ea8c3f4dc1fd130ff175d306ce39e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526818.17/warc/66c49c88a1df8ea9e315bbe5f20743125b9bb132.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526818.17/warc/91bec5a4264529dc9138fb395501e924d66167a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526818.17/warc/b37bcc42a9fecf525ac93f8229ded1a73d32f726.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526888.75/warc/38ae1688c7134c43c12fb09cf7c0b096bc2c2e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526888.75/warc/45f3ca02095426d07051620e926aa027055c0c0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526888.75/warc/5db00e19be46499b37bd702261f14f1d8fe1aa8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526888.75/warc/ba81872d79fdfae4ed57f00e8658ae14a806df7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526888.75/warc/c3d762bc35ac251b99aeb20919a4746b906fecc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526931.25/warc/4d72644cfe6121a2d83b5682e48239ab3ea57973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526931.25/warc/573dc18e4fd658db094c9415b14b1afb7ba1c870.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526931.25/warc/7d5731fe6d393c17119f0c4f3a29c94e6cc50b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526931.25/warc/7e5dc100cdfad151f38b9b1e1543d8499d1c37a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526931.25/warc/bfad58b88a39007d34cc050c16ae518861cc2cc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526940.0/warc/286f04a3995a359de09934b837e3140d337b9d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526940.0/warc/54e7452458f7e872b467eb4e3a7cf16e30139c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526940.0/warc/c00ee35401edcf536b43a4b0be39126d63fa5ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526940.0/warc/c8e746297a7bf5a1336e615b3a63c012367060b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526940.0/warc/e8eedc1fbc0c3ac74bd987e34d4e9512bb94cf8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526948.55/warc/12e3efd8dbdb644333263acbd074aaf8314769d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526948.55/warc/252b6fae1b26e59f462e6ceb3bbed09119aac71f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526948.55/warc/71f1559c013a149928fcb8ac9939617e912d8886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526948.55/warc/bb8496b0feadfc9e14730e11617375e2e578aede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195526948.55/warc/c260e0f4f278e5a02ccb882c5792aee1c9cdbfa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527000.10/warc/08c2c18701e1d2c310caea1a0cc02acebafdbd24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527000.10/warc/202e2da4c8d779a041a0e3b9d54c8e5c31aeffb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527000.10/warc/53da2100e6bc6c408e79dad54d6ac57bd018f71d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527000.10/warc/bde1c96f01bccd09e2cb9e2b8e38836c1c0f248e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527048.80/warc/48298b9bb68442d8f11830edf390fe5942fd5985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527048.80/warc/920383e9c4d2473d10ea61d6f95796f666afab62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527048.80/warc/ae319ca84e571c319fe7ba93157e5a86764b0f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527048.80/warc/daa2e9aed93cadbcc2fb810086f2cc26a3ae9a40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527048.80/warc/f981f5b1798659d9e15a4dabed0dfbbf25966121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527089.77/warc/4eff0952023e2f6b421615a1328428515cbdf8ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527089.77/warc/930434d19dcf1ca4d972e7d577175a5fcb9bb082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527089.77/warc/bb7a156408690c8e86854f11bf92821a7521085e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527089.77/warc/c6238f86cb00ee44f9a97c0feb940d2c8dd64a2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527089.77/warc/c95528afee76c9a383bf934ea738a2457f5a5304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527196.68/warc/24eb58499565a51ac401479a602e3fb90c9f487c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527196.68/warc/98da10844dfd0880637a19e298f6702aa5162d71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527196.68/warc/b730bf6f57f0a654f700336efd21ec729b0fdb3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527196.68/warc/ddaa4ffe0011300675c02e206922785897552ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527196.68/warc/f132a130403f332762de0e8728ca0b050e21ddc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527204.71/warc/545470103573482218834e85bb7725785b078105.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527204.71/warc/9810f415df6437a8317ea5b0fb8ec3dbad26e817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527204.71/warc/b4542416de5a914b9d1255defbf3780fdcea5db6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527204.71/warc/e3ebc378a7399323b50ca0aee8a949f88625d963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527204.71/warc/e58dfc4643f70da500539825776b6f14530edd6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527396.78/warc/641067c478a74d1ff6b21ef946984a55df93cc07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527396.78/warc/806cc44297923e4628669c2ce0a4665e4e4d84df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527396.78/warc/abacf486148779eec46f4ca2af30cebae265a93e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527396.78/warc/de98eca5dd1fdf18755912efae7a7c6556183c9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527458.86/warc/03f76a6fcf87b9d22b19bf63e24c977fcf9647f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527458.86/warc/31fbf1878431ae6f6d074c5d50b5bc8cdf32b6b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527458.86/warc/3dce4ca06f33429a446ca3ff14f72b5b50cfbb65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527458.86/warc/6dcc82a9e2ec3eee1114c1d8f08c5e8fc470e550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527458.86/warc/76a88cdcd99d3406732a8e61a0b8303ba9a657ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527474.85/warc/71ce9ad0ce45c0a319760ddadfc4ff27e6f87cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527474.85/warc/af356c07f7eaad459a74e1fd7771cd71ff804b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527474.85/warc/c5cd6434692c3acb24458d9c6d978fd788e25af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527474.85/warc/d0a1e7db620cb30a5b43e751c6d801491517b047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527474.85/warc/d22afcc7c8d489c1ca869efd360b3e3aae81b139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527531.84/warc/443a34146fa772571f0e5c111fb53824a75628bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527531.84/warc/6575c9f353ddf928ed04ef140eea267d0687b1ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527531.84/warc/9d896dc213c1c59421df9d755d517234834c896d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527531.84/warc/ad9c177d599bf3955fc48ff9b3569365ccf1338e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527531.84/warc/fa08c1fc19f87683181dc0c2ca1595f13def1784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527828.69/warc/151acce261115eec8b4736b35e226e3fe291ba60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527828.69/warc/461f3194bc6998e88275d9402f4ef866530e97d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527828.69/warc/7912f70b742e1fa1a5221c4b63c789eb19527c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527828.69/warc/dd17c8bda79d4a5e057e9479e915832eee18cd1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527828.69/warc/dfd7d9be766d673cbc4ab4f96eee727d41cfd86e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527907.70/warc/258fa2bc9d25f3f36236c8c8d84c738e091b89b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527907.70/warc/25967a08fa635443f8b69d703088ab8d9b00a165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527907.70/warc/6875405141524a6afe66598fde53033da73411ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195527907.70/warc/bff2227370dd4f1d7e4e3d22852c5ac1821c568c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528013.81/warc/11c87974f509a29f11a6ffd9f3a04dca0d46f894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528013.81/warc/395f61408e4d6ad5239afc47acc1d7e8a12e3128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528013.81/warc/793e0c967666d9541cacf65909a084a2fa47f71c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528013.81/warc/902010193b0562a409463a14c4c5933efc1632e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528013.81/warc/dc434e8a1f7f8ca8510c7747eeac9efeaf80c431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528037.92/warc/1dc0387ec37cde4d1f1197d55e5fbe01c0d00a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528037.92/warc/7768820dd94fd2a8c9a3a6bec3d28359abb3f5cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528037.92/warc/917805d9a561334ffd45a697a52260bc1018fbad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528037.92/warc/eedecc926710e21dcc4b0983537b4d450d650a1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528037.92/warc/fc0589d466a90b3f73284a8b2268239aa9f19160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528141.87/warc/5ed7c362f6d770e49caad4f4e5a6d11d5f703bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528141.87/warc/8b7bb29c4a72f4c164deb429195f99150a164127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528141.87/warc/bacdc66420439b55300dcaec41b76f0d8e35a491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528141.87/warc/c0b68a9c09041e1fa5927f882589faf5f01e983c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528141.87/warc/d2fb9ff2d6f1374595ea5dd5bba7cac3142b08bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528208.76/warc/0ed61f91fc1c3e5f7e2342b81ed2454c2cb347b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528208.76/warc/16e1f571984d809697a672cdba70e5e926769dbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528208.76/warc/6485d2d59634434116d9e8c4e3f0353da801355b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528208.76/warc/e42c0e6dd960208b84482a32b54a5a6ca9727a84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528208.76/warc/fd9b16e2014d3801c2b0e5b83c2583c43f720dee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528220.95/warc/0fc0291668bdf87bf01ec0830f1da3750e79f792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528220.95/warc/121bc1aca76588de95fade608d5c0f6cec40864f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528220.95/warc/16bff081f686ce5b33c1034d764d7c12630d4350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528220.95/warc/77f69e6dafc765896df66a10fe53c1344484f00e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528290.72/warc/34407ae8ae444bfa76c3dc22097de714db0b306e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528290.72/warc/48a38b1b4d0d5511c35d85313cac734a2bdb238e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528290.72/warc/67b9cee104a7378e795499096589fd770a73919c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528290.72/warc/76ecc1fc607e7f529dc22162e1205d4b8c1ac6d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528290.72/warc/8df8202f3caacdae5d385a5966372ab0c722552c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528635.94/warc/3f253d000d7c2f3814100a44598a83498c14592c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528635.94/warc/5b6e1a031658ef74d0246b0f23c2104271783ac1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528635.94/warc/d3b15a310a8415d15ddaf1e35db2061604e1bfec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528635.94/warc/db1d66429ad487f0782fdd37b148dad88832a502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528635.94/warc/e4d7826f5d3fd6049d64b86fb97696232e15a239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528687.63/warc/0918bba461cfa80bcd39b9f2d095578d6ac9a442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528687.63/warc/3cfdc6fcc4d902c82ff2b7de5391947ae0f41c61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528687.63/warc/63134306b67c59a7925b3a5c73f1aacb305bd273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528687.63/warc/7eba27cf328de16d5bcf77e088e49f353baf5d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528687.63/warc/9c559b6daa4f5707a35d64056ac7fbe648561fba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528869.90/warc/2e7e9a5aabe790b2d73b5c3a15011f4a21ad3662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528869.90/warc/79619f38c973a88f4037679a7b8706961f691ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528869.90/warc/bf2365c8ea74a8e2d4ec6e97f1604bc6185010f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528869.90/warc/c2ea12b6bb5576629f63cb620418677d5c1cb6cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195528869.90/warc/eb9c6c62e0248d41a96843874a0671a785caf49c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529007.88/warc/0e3c07e687831e85af99b066de3e557fef3bbfdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529007.88/warc/17c50519ec4b520422be648eba7c85c13ab4b5e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529007.88/warc/37ab8aeff58fb7e26629fc092af8b1c94cbbc864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529007.88/warc/a6538d31f95e058adc5dc0f886a73141470651af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529175.83/warc/21c12296524b96870a86266996c1e2bcc60720ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529175.83/warc/77276b7db2c21579fdafb71d8b3c986e364843b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529175.83/warc/aa08f43c1c6bd8b319fdcdbf358847598dc7e991.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529175.83/warc/ac495ddedae31d187fb13dd80a0e9de6e42868a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529175.83/warc/d0549a347fba68f5e20b162ddf60b12a724efe6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529276.65/warc/095aee48745dc3d15c12c4173d83344b0b023570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529276.65/warc/5aba847fff28bff40063a88bdfbb4ce6b6343a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529276.65/warc/90761a9e9f04ec3c0093ec9fa31fe6a1fa711811.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529276.65/warc/b49da325c3971cdb4728e509bcf638b4ab4c39a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529276.65/warc/d59ae8c67acd8f08a189c21c8674c9d7f4696329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529406.97/warc/358458ec03627a387868cad2eadab41eb9f74de0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529406.97/warc/490608345d492095476851b72b26c5383a49edca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529406.97/warc/a73030b7218cec09f321cc2c594859cce92c5f35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529406.97/warc/da4f435f767ff2354b1e41585dbe9cd9519395d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529406.97/warc/f9899a98f89e1f087649f2d409dfad0e80d8a89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529480.89/warc/26921454b162a259c9d8b98270fcae9a49ebf93f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529480.89/warc/990bd5f918e9108625b4d47428f80d8f42bd8081.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529480.89/warc/9d6df5f34f1d614721e4385cea8355554fb90a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529480.89/warc/e433c06cde3c31654c828e671fac9ff78668ebe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529480.89/warc/f7e79782992ccb9a368e4a14c5c62afc31e90eac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529481.73/warc/1b8cd0d4d6d6643351afe3ca172270df18312a7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529481.73/warc/717e28beee0b869e5e70da7dea534dacf20ef964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529481.73/warc/e54d7ab2519a73cd2cd59d6cfc50afa06a6f7d5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529481.73/warc/f32bc40db4bc4c18bfbf76056d95de70379521a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529664.96/warc/5baa99fa84d5345cded85eba56a1e4f4434e09f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529664.96/warc/5eeecb24ee29975dd3fbf67bd95a64658cdb3bc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529664.96/warc/76f6f008105e3ea62963670d34bbbeda8d0fd26d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529664.96/warc/be6c074ba18f70fd225940ae188301e142926878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529664.96/warc/dcb53e906ab8e5829f356dce86e159f23bb07dde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529737.79/warc/417ea9e1241722c2045954e77515e010a3e599d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529737.79/warc/48badcc358a5cbfa2071243f5c6f27a35ac8b70d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529737.79/warc/5d7a0b4fcb2ca4be02b08366401db496c7bd4cbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529737.79/warc/68bd3258a1528ca44cd91ab034f07ef1dc6bafe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195529737.79/warc/79d3ee084024dea1e33c84d65a2b1dffa49ca87b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530246.91/warc/51202b45edf76dd69f5565ea6a927fba5b326759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530246.91/warc/6a4e7a02d0ede5b4368e5eb9587c626a77b85da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530246.91/warc/a6919da1061705a4386bfc4d12f7d86efa478b89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530246.91/warc/d05518148db4558301719a23130ba301d2065eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530246.91/warc/ffa2f282b03c6f4177a0d57d4584b61355a0621a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530250.98/warc/1be6fbebb5d8d4199d98403b05654a21b53bcea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530250.98/warc/85ef1a5c8d386b4266cd5af31f67c070b5652fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530250.98/warc/ccc968bdf931c586ea6597ca3cf51d4d98484c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530250.98/warc/deb8582b40a2cce09d0dd8cabeb09c26c2e45538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530250.98/warc/f613df4e013433099804165ebb4e408b925d4952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530385.82/warc/37e41f35978c7c854ed2cf6bf512c6f8ed4c80d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530385.82/warc/47ab7bcf4d38d32830221bd3c1460c7954801e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530385.82/warc/8b331f8252c2b39e16fdebe15dfe37581ae10fe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195530385.82/warc/c097d13586b9dc03975fc7fedff1ea590fbb5733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195531106.93/warc/2561d27451e3b2e9b94d90e77249f1e530d2085a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195531106.93/warc/2c8f4adfaac88bb591b7397ba6262496b1b9587b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195531106.93/warc/3696158ca81f22b71bbc163a76d5cb797bfde47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195531106.93/warc/df9988516354d1bd94da61c07265d57a6e03328c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195531106.93/warc/f829962db58cd9d7e8de9787f7903808c0c1719c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195532251.99/warc/2c8e932c69a1a95314331164bc726270568a89d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195532251.99/warc/3c1b302ff226dd74b3a306895a032190228aa4d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195532251.99/warc/b67378435bdce776d2f87d33710dc681b74c0cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195532251.99/warc/b97b2708194d5e4543c157f12bbe6c9736174ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1563195532251.99/warc/edcab92625a7f184c344d70a7c4cd982fcce5005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/1af3f49a89f0551b4a1f20b209f8c5f295878f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/2118002603d6be139e6f989423055accf8c5a5fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/2497c4cbbd7eefd9e3b77755ce6afab010efabfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/25e50c76fa18446843f317fcdacc524d8cc7082f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/26f755b138946b952981fed343166c0584fd2156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/2bc64926a1b664607a617c76944433d2b3efad24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/30845028c60b257b45d61c12574fb0bcadcffef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/31e482c13d5e938a96a41324f40b5252649fe568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/32e7245a68f1d0a22783650853facdf3dc485b56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/339fed8cb0e9d065ffdc827dbe220fc1bfc1f84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/34977ebd28d65d91d4f45e6772b9bc19d7c3f5ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/360d91ccbba6c87235139f0327e0b6604a3c55e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/3b8e4e02b58cb8520fbf211d1c5e597b6ab722c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/404df100a3d6d386bfc267c8a96e4c5ef2106332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/49166efe9dcb659f55367a4ec9523f670fc3ab30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/49cb37ae670964e061f54042f01623fc5f66fdaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/4a6aea2df3b52ca0563319661e95c7514063a768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/4d6f938e0b6bdc754f0ba47c3488c6ac966f42f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/4f5515d94c390cfa6167a3ba39aaa444b16dfe08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/5116bd8891271b8cc9e497cf33e4dd37de06b604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/551d788b0105de3b3d21b3780e65ff169286f432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/56f3e39c3f889bc4e2cb0360183a7751b569e551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/594868336d89d1017d022f01a231b7a348e01416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/5b22a3c6a515c455c3225a88f790d286c7f95c80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/606003e94f562cc605a548c5b8389e79fa2f97e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/63b75178669b54a11cb731d27ab0cc36aacd6da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/665a104ecfe8a1219adaed8c661200576d251a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/6ce59e6354de2068ad4a2c748703f2046f7aff09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/6e736755a75c8c9a552fc952bbac8447e86f4251.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/6fd578a49b2df4a9526c219ad65b6bf6c6c8235c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/70589026c97b1b49877320a14939f284805a5604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/71ca598d2b362286f8fa4af15c8ee0859cba6e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/75c811e5420750f0d14f2f456117a1d99326b26f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/78c7ba37ff1b1e333953f948db48811c3cd838a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/7b6e6ed97c47e129d33bf6b4022d8496a2dd84ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/85f9454ea9a65dd7feacd0a14b56ffc06cad8eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/864abeefddbb5e2bc6e3d951741cc0484ef28d3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/866a4d2df4bfa11b731bb9c17a9ef6992ec5ab4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/87ba26654dc173ca5ac961b5cc990a33b7bf6f3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/8bc6a5ce44b2d986aa8a4bc22f24401667c3923e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/8c5557ffd856ce23dd1eb177ecb9ceb2f21ae599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/91cf1b8816e7b4f19785e8c8185f0b0e5586ab1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/9a1b096090a1f627309304ba4d9cbc9cc3cd3da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/9be2e811ef5428c3ea92c47217cfc3e1eaaee7f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/9c3a8f77d19445e59f38eae0826cc459cf7f4c2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/9e87db1b3d12471c466d94ec98b4274fa6840a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/a17cf2703669bb141a528b6ea0bb94ba87cf6e63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/a7ffe60c4aaf6df347e60a5ea8c8a9bfcfaefac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/ad1d8ac6802e582477fa811c03df48a57030668d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/af1f6ef38b3dedfc77cba29c91fc56bbe7c9997c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/aff7401cf350c0dae67d4bca56ae672182c46e3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/b4a6bcccfa42cac5344fe5813e01af8669fa0791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/b4dade8801a7e0803052f4c358d51ebf17091305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/b9b323d5e15c200c30215b4348f07a5c33c911b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/bca0026af998f16cce54653aaa12ddaafa2b2369.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/c45e09ef278a5a5a2720348b37764b6947c8ee27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/c9737bdedeb1322c71496425cb8882b505be3181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/cbc3f619c36f56786edcd9d8ca8fe36519188a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/cf1b4213aabf8349daeda94866532f077d68294c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/dc0a5c0976517504e6384b2601ab585fa62f71d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/e0078b0c1fbbc105d97ca6b1ac45729d836a5e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/ee8aa1f353ae2ea5ebf8afb08c6267793d659097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/eea680c27cc2e54378aa8f7b05f0d763fadfeb5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/f535e02273079552192770fd275c53172e7e3d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/fb13ff6f0ddda858b853de365166c27231011a17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/fe0e09b53d890e5d82064b1c216a56944a7b804e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/ff2441d6895526be54fb6825bcb836e9109facd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/ff5a46fb588689c2c1c6b95d415cfd0312d5ba3f.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-30 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-30/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2019-30/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2019-35.yaml b/configs/cccc/mixer/CC-MAIN-2019-35.yaml new file mode 100644 index 00000000..bfd92d73 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2019-35.yaml @@ -0,0 +1,674 @@ + +streams: + - name: cccc-CC-MAIN-2019-35 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/023577e73189c373dbbf831711d04eb43a27893d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/048a320b5f2ca6181a219c039f66141ee6968ac1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/08d106ff1c62b580fd69a05ccc886dd6c771c432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/09b0af8ff2f88f756ca1b1bf2d7a8b080fdb9053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/0b427afde0da636f1c765c1bb8d0c644d8162a7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/0e29dc9f05c9cd9a7db5b6ffba7678de8d9f7f43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/0fd04f40035971dcdbeb6e6fec3a5b1511bccf29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/11ddae3e014819341f39e2c64918c2338a9b1980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1527f24569b74fa21dd494d8a66c7ad8bd198a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312025.20/warc/128dc27d42904a7ea14a0263f053c4aa03ecc2f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312025.20/warc/47b07e4a8e12da3b1537c2b85f0cd73ac878a144.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312025.20/warc/772635b359081fba825dcca659ef910725ee0d6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312025.20/warc/a135afa3ca9b40c3d2496fc6f5cd8c4a854099a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312025.20/warc/b430e529d7948f806a05da016b8113e5c47e722a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312128.3/warc/6fe46a1a118acccbfaec56c0f0d05f8cef7fcd60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312128.3/warc/8086b4de2f1d179e1148676a3bf2c8d0cec1f7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312128.3/warc/84396525621624a4ff4ea2dc512045631eee308d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312128.3/warc/b47799dbafaf43d0e6f25ab33705f7711e4dead7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027312128.3/warc/f872549dddae6125bf91438082e7bc0d66c366f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313259.30/warc/3336394835f54d5cd1f353d3e6f87c16eff2a6ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313259.30/warc/dba655908ce2bc8f09e00b5e53a46a1e499ca691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313259.30/warc/f32ac2fe00637c3153184914362aa5eb0237d7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313259.30/warc/f5d109925a780aa0751c6984ca349d9230917b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313428.28/warc/292ad5961e43f4f17545616cf2123a1435c9bce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313428.28/warc/3d23d0711591b7ca89e6cd82578dbd25f874a3c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313428.28/warc/59f3884220fac95c71b0814f6e21d87418fb0004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313428.28/warc/8870f231bb2dcc80eb0620626c4fddb0a53e41dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313428.28/warc/d13011fcbc05f1497caf2701c105358c45edc6a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313436.2/warc/106dd3a1c4ae4e407b993040f76bbb8adf22b92f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313436.2/warc/3e0ac75d54f5c450286c53a8fc222fd16a80568c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313436.2/warc/9950f5c4ed4dc8c9c0eeac85c103e38b43783cfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313436.2/warc/dcfd37f34ca8aaa9a03d9901d734c4543299c6fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313436.2/warc/f286cc8337a8febba4ed1ae7a11874a116b83de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313501.0/warc/8d611d3ea7a9a500c1c374c319cb8e624e7185cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313501.0/warc/c2410d00227432f971df9737d8edec98e7ab7f11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313501.0/warc/e587a4dcc9ff1634bda661710d75dd567589dee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313501.0/warc/f8b006c396454e7c15d3c6fa07c2741338c1e2cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313501.0/warc/f9ef686f880f11ee181f84b17110462d62e545ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313536.31/warc/32e28d2f658933e08dd589ac12d1cef39940db98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313536.31/warc/49cec5f8aeb46d3ac2586df9cf90fc26e3c4deb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313536.31/warc/586a0dc1d4b66f9cb5ca2525072c5d03450469c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313536.31/warc/61b9b6b5971b5125374b53f97863de54c0937b96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313536.31/warc/6b5b276d0eeef761fbdb6301a08aff54df3691b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313589.19/warc/36613ea2cb7d0c1f65c0e5c4e972394b2b586985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313589.19/warc/6d04bf02a035be6aae1885841188399e47be6a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313589.19/warc/917e7e7374bc381b999267152892886c9dff7d75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313589.19/warc/b63f39ca87e5b43a5b255a8eb1ef1f136a626224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313617.6/warc/2121c07c93cce341febe015f6910be6ea499a07d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313617.6/warc/42b07032c5d8c063147b564dc4ad33317439afc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313617.6/warc/5fbc0c6d738119d6bb39caca1d10d448612da1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313617.6/warc/af24135ae0f0806f751175c3bdecdb96c5b2075c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313617.6/warc/dc2713fb71d081a40430b8f4e6a16a1b2f4cc953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313715.51/warc/3c8de09070982edf289a226d575e3357d2cc0094.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313715.51/warc/7e9f7b7f6fcb7e9dd423d46e85393c257b707d30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313715.51/warc/92a32fb0ce131967416d6763d2b8dfbe457f03d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313715.51/warc/9d95be1de3985a024e6c5db7b97c46cbb5895a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313715.51/warc/affecdc92f7ab2bb50268ba24a2233427f82a3ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313747.38/warc/4640a577a63faa2f91f838f2031751772f8be68b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313747.38/warc/64700fd4c4680f7932b17240c067f296fef18243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313747.38/warc/af30ee3773436d709d61dcdae8ff15a77ffd5b58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313747.38/warc/d0a1c89b33fe36847a10c20f73979d7928d80a86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313747.38/warc/dcec005507fcd7932d0ac9ce41f1d147a2d1fd0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313803.9/warc/13630c7d743a6f1c27cfbd37888e314b5774fb69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313803.9/warc/bdd013fe204d52973f23ed456930e015fde2d521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313803.9/warc/c7c112744db431dcef4aa412fbd9f4c4b965f6d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313803.9/warc/d68004e8790b6e035b141b2fde61988bfa5bcf19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313803.9/warc/d7bf37498c149524507d231a48279db420c9698f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313889.29/warc/7973743270eb90026286ea90fa04346a977718ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313889.29/warc/a204886543c56bf92cb48c8b10eeb5224c3998ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313889.29/warc/bc8d287ddb1cb032b304464c33bcca78ea27d867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313889.29/warc/fd3f23b86a48e0aa1d54258b45b243853cca63ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313936.42/warc/1975bf1979aab11f68205d9c37f2d03074ebc5c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313936.42/warc/7cc2e6d2edefca3d5067db8583ce9f02c1064b90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313936.42/warc/9bad16b2fe564ef838814fa91a7d08ce276aa2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313936.42/warc/e2154eb97db667dacfc0f170a08f4c19e3e1c88d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313936.42/warc/e82fce01b6f4d5841142ef782e1899958b33b39c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313987.32/warc/08cdf99690d2df5ecad2421823013489528e5a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313987.32/warc/0e5a41a98dba3bb5167a5941591b1a886e885cee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313987.32/warc/5665ccf592e1c70b3750d9ed3aeb01a6f60c92d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313987.32/warc/e326fdfe74560712bababf1574a5fbf67eb727e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313987.32/warc/e6f241175470b019cb437887cd2d5dd47d3f1bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313996.39/warc/8ba2e13979ef07df62b9927b34804faee2b88cad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313996.39/warc/a60cd15538d0f467033adafd1f817cdc8bf4cd60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313996.39/warc/ab4652ca8322fe0624065d6cd05bea7a7f325d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313996.39/warc/c259a185e2857190b1149c52aa8aa334ce3c840f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027313996.39/warc/e34407df989700ddcfd4ebafda24bf83e3d6d2c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314130.7/warc/409b06e7a5a3dcc9c3b8d4395973a29527ea4289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314130.7/warc/40b37b53a79ca14795098d00450081fbbce68a73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314130.7/warc/a947cc33ff6d91397e98e85e272813a6e849cf71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314130.7/warc/d9c2a00c7eeca504a8eae8d554bdd4c038389646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314130.7/warc/e4c83da711d2012d26f14ddb2eb105b796cfe246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314353.10/warc/3c8065c4e0579745e61fefd47681af1807f055c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314353.10/warc/80d7f231c5cc6c1731b66897f9bf251ecb35dc5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314353.10/warc/c66bf390854ddbc0d3b594f88c583caa736bb485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314353.10/warc/c999e69e141bfc72e7ec73291ba35f49358918c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314638.49/warc/23db96401b9923d38ebde66a7bd75109dddd3e0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314638.49/warc/294018bfc73b140a69e9aec53e52a743e352065d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314638.49/warc/84a7e42836a382d287769dd341b72c61b0e4cf9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314638.49/warc/c8aee7cca70e2c7dacb228996bc8e697254caf35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314638.49/warc/cea69b6846517c613fe60060561bcd16cff72123.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314641.41/warc/48379469f4e53a45c40e6dc5c2844e4f7b68a522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314641.41/warc/7cf83cd8685e04a9de9abc635d28c680816e61fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314641.41/warc/a412f4933a70d2e6856e53656ab4d012bd2ebc87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314641.41/warc/b6deb822ba2d73abd02f549c31cddda3015c9e17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314641.41/warc/f818f028e0d0cbf2fee5e70947d1fda3017624c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314667.60/warc/08a1db9a7a0939367128e6e6e2b81804c66b6018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314667.60/warc/897bf0c103083bc117df662dd3c1d76bb6eff497.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314667.60/warc/93f712f9abbd60acbbbe854910b2c75ad666704c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314667.60/warc/e814ab889393ab69c5c0511cc17424474821a2e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314667.60/warc/e92d0ac3f4a42c7711740edf56f445250f6cc174.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314696.33/warc/11c741616487685251dddd3713102eab79bfe679.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314696.33/warc/2849186d3a79fd63cedfc1b9d98ea7d94d126b21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314696.33/warc/6ce40cdb7d37931b6d017ced7b62f17b924cd674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314696.33/warc/c691951336e6119dfb9feccf02526d404e47e373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314696.33/warc/db8e47fcc55247babb2426d2f5a7594353a82e4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314721.74/warc/0ad4ee4a8702e4f5bb6d652c585ab5743961419d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314721.74/warc/0db5c5c6892e5286624479e6b96c09291f3f227c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314721.74/warc/474b9a032f6a78f6f911f555c30ada52a5cd6b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314721.74/warc/8d95bba858c4309d1e4eaa9dfdb992a94210ab3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314732.59/warc/7755b538a90d8741daaeddb925875ab864a9b344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314732.59/warc/bb1261809673f905e205654ea429b16220b38740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314732.59/warc/ea352764f0d66bb49b0465416c4e5427efdc5bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314732.59/warc/ec126825331fd77d2f1f49f5c7761eafa0072f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314732.59/warc/fdbdb813123ecf7762ab2511e2d19b1ae767ee74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314752.21/warc/4434d472cabaf0df8b688a9de385ff1122394491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314752.21/warc/54e9175982db4f37fe96b343e485a04362c0e1ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314752.21/warc/94d9d97af73ae9cce7e54e9ca66b65e7d9c02d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314752.21/warc/a64de1328d5a74398b3185a074986d6d65f4cbfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314752.21/warc/e8a81242ca236ea217f8bbec092e26570e963dcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314852.37/warc/1fa9aea16b861a1e52cb6ded4e2dbb95bf737133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314852.37/warc/4d98aa45b7eba3552123bcf07c688ee6d3e1b647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314852.37/warc/5d05b709c25178cf79df0187f2e38c2b03bb6ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314852.37/warc/9c136806cfcd1116b5621388b82da78dc5007bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314852.37/warc/bab02b5e42255c1f166a5ff3e3a69817de70f9bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314904.26/warc/39622fcc3fa3643694599dcffb6436656ee73703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314904.26/warc/ab90c1426ee605adfb1fb40cbb44ecb4291f062d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314904.26/warc/ad3dfb6a3cbf7f25e0fa8bf53b11213a35cd61dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314904.26/warc/d360328b63d60b667c8a52fc1b1faaa6f78678bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314904.26/warc/f93a54e478342b9914522a9229dd3a212219fbc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314959.58/warc/4b2eb859b1e3835ea3ce78533c36bf944d4d86f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314959.58/warc/9ab4a11c7b586999c609f589eb5f91bd013e05b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314959.58/warc/dc0ba603be082ce3d07fefd33038ae59bc736125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027314959.58/warc/e24ad16ca55cefac30101d39116248c814b2a75f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315132.71/warc/04457c4eb7fcc9b7beaa93764791e0ecec48471c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315132.71/warc/3b401a8e5c613ec27ddcf768cc748c95b114bf31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315132.71/warc/8bda8ddde1b9cbd62bf6cc15f51e1540a907aea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315132.71/warc/958b847dde8ba234ee253d6d643e52b371acfad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315132.71/warc/bfd916d8d4060e5d1c40c7fe9337c13ba88886de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315174.57/warc/3adcf1eac8ba991bd33931b2d8074779cb0bd5eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315174.57/warc/457feaee971fa3e64c94175bcd1f6a5a3c985caa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315174.57/warc/91dc3360b34cef9799412748cee2ddabcca9c86c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315174.57/warc/a34076c1958c42de33339baf1c0b60b020c3a141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315174.57/warc/cd61d63c1ae4438fbb86c02072297d591340b388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.14/warc/0698cacc325d925464af37f95ae2426e2a95ec0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.14/warc/371819f08e6129b18a66278ea545f37199c66dce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.14/warc/71ba1e03d4531fcd4dad84b098eeef4e61e28326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.14/warc/95c4004f3636f4657fc549c7c12a72c58f5e8d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.14/warc/a681d1c6425bb7d53ecd2f4d0fceccf200f59b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.56/warc/27568ea0acc00e5763fbbf2fadffcefccc89b201.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.56/warc/6570478d5e8e079110bd2683eafbb73c09666fc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.56/warc/8aa1781b5669756f90ddc683de1c07f8fc35e2cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.56/warc/ae225e902c6658b731c44422564a475f4dbc3431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315222.56/warc/e873263e3e0c22773505465b99f7219991b35472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315258.34/warc/0988eea3117198fadf3ee51f1025c76660bb2f01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315258.34/warc/4735f4d5e86f8eb8bb7d71594ef5b44e1af584b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315258.34/warc/7503959b87196d0565c2ea4358fda66768280066.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315258.34/warc/78ccd513c47e6ee75aa207c2e8aa58c57da06d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315321.52/warc/1ed1c62c22743ab85898f9a6d159c6e7dbd26357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315321.52/warc/38f29ca8be31dc1575101599af51fdae58aed378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315321.52/warc/4033f64f46587315be13a08af04acf8b3e289419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315321.52/warc/4bb5239bf0e11083c41b082be58cf824fa51f39b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315321.52/warc/601a5c8b4fcd59972ce2844269e6efa171656261.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315329.55/warc/2885902d84e9afbaf85694d4d21e5652792c2406.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315329.55/warc/6409d854b4216417efa258fd8376102651024176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315329.55/warc/afcef1190ef8208db3741ffff9c162b03d396c17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315329.55/warc/e39e044cf57e0281a0b254dc99ef420c90c96a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315329.55/warc/f8fc9260002a5e29991f0ed31c668c3a3c5c61e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315544.11/warc/049f31c2eabee5577bb33abb81850afc21d82aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315544.11/warc/5116a5db356cf7da9670e0167340e22b08d3d016.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315544.11/warc/75f680485e88a8a599abfa4bc9c862554e362800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315544.11/warc/78dd0718db358eab27cdd37b0e8be60ecd744740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315544.11/warc/7c3fbf598243c0286838ab02b2c5d83878349a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315551.61/warc/0370dca89bef365b8bd7afb133a9c0c34e449472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315551.61/warc/17fee55509f89971e2f7e82ff16f76a63e649a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315551.61/warc/3f5260641dd5745e3a55da5cf90a42708a4f861a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315551.61/warc/6a2ed302ade3c275c2f7e4f90f4a7f9281727705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315551.61/warc/f8e00ea5824fbb052f01a554afb4b921efecbd55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315558.25/warc/3a9d4e93e03ab60ee304c42f296e9cf38f9ba746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315558.25/warc/488b492ff6326ac6db924715d4196fb2b07e5318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315558.25/warc/a0c83c142da022e91804e2245f7174a8c531d265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315558.25/warc/beb7dad337a2a79b6328cda8b7347b12ad69e590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315618.73/warc/53ea3fe82de88bb92377281c12c2bb84ab5e15e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315618.73/warc/75b26d959101ad193d907fd34a729dbfdf1b0666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315618.73/warc/b1bc80293e4141203a8f28f93894590447b8ecb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315618.73/warc/b842c32d79ca37ab5674d07c3d580212bd0efd4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315618.73/warc/e0d269fc63bc9549bcdf04e939b3db3863e7996e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315681.63/warc/26b5e08b759c9c4b606b4f3de9b9907dbd48aa30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315681.63/warc/a4709b28d37735325cac8a8332b6e4f8f9c8c099.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315681.63/warc/b6093dc0f3ddebc57061cbf4a680018282c0167b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315681.63/warc/e0a3b71c78df410e8ffc83f66cc737c3fee59281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315681.63/warc/f0457c73f2abdd9e8e66a5a2ba7693e188965f8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315695.36/warc/34c06563b580a1c9c565dde6badeb2beb92db1cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315695.36/warc/8209b2424e4dbd43dcb044a414e5f5d8eca7cab9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315695.36/warc/86fa7e7f6f8f0429b4ea2259232727f4da293d50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315695.36/warc/99c0e1a5da1908d544c462b295247bc4e59da7b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315695.36/warc/ae2d297e6619e01267121d8f7f651b8ce11d3f20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315750.62/warc/50108c02841086b811fe055cfad9529d5fd91c6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315750.62/warc/673dd2792d4fb8bc12070b7a009c6b6e74f0a20c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315750.62/warc/8bdc5011a4d520dbd699b660da6fdc809a6e0a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315750.62/warc/c76d10164b2bd7c0ce9ddce5fb26b44888f4f5e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315750.62/warc/d3d1307159ed6edfaf16493c559b69621697b3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315809.69/warc/4e0e5db80d39203c8436884402b963203f6c5dbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315809.69/warc/6f53232f190a18d1e7b8623fefdcc9ff6161453e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315809.69/warc/c7e570a6100504ba687e76bcc06416b56483b544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315809.69/warc/dcf28054cff055dddc97c645c0f5afe61a59ae1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315811.47/warc/43b6348335504beb888579398f17b24df35f6aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315811.47/warc/5f6ff7f349fe34240f2de695e5102be8e16a4cca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315811.47/warc/6d6868dbf3acf02b5dd820f09c3ffd16f4e8b009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315811.47/warc/7411193c3e8c30af1bb00ecc30dfcaedc383a8ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315811.47/warc/90b0dcd993b18cc8cc819dc2e07c93829644774c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315865.44/warc/1c83d87fe2d81531c5ee96ae374d72c894c00a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315865.44/warc/802e0eb2fde204896550b81d9e0b5ff1a87a205e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315865.44/warc/c35c951f6ef7ae20f1621aa66b9cb16bcd66963e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315865.44/warc/c56a1a1967a9fcbcd83ac441a4232a8f6c990570.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315865.44/warc/f6c9eb3d88cafbbed5a8846929a32fd55d1368df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315936.22/warc/04a17b6d9611382bb42e0a8f62894d565068297b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315936.22/warc/0515a0629bda9f6d3ac08410753ca9b672ac3959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315936.22/warc/524674f654b398dec3cd147017a9fb3d98390cd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315936.22/warc/b08d88de2decfc0a70d79958d595e7eac5e4696f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027315936.22/warc/ed8006a1508d1fb8a474ebb9d48e2f9f0626c325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316021.66/warc/246f863f75b77e878441f732e857a6acfdc0e736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316021.66/warc/39aca1b299dc0c52fec84a56741445fba2ce8d79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316021.66/warc/aa5d984f03872e7824c86d295cd090d0166aa521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316021.66/warc/b8f84b57cdcfa59767d1040ac3a3832b1ffb3ec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316021.66/warc/f9abdc94750139c6c8bd9dd08577673bbe26db00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316075.15/warc/6de9620a6e08f52b6efdaa457c8ac3b6b2b4a164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316075.15/warc/9bc8d1a9156cfbb899ff51807e410378c3eee3d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316075.15/warc/b1ce4b09fbfa21f530851eff67c82dedeada7a34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316075.15/warc/b92ee0342c50987f9aa5a197ee9773064706f4c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316150.53/warc/04d6c03e307e945116b9fc32bbbbfcbfbb6e6320.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316150.53/warc/3e2cb1c3b221013e7d35d11af22f914a2eac368e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316150.53/warc/41e0621c6fb4c344b43e522ba235d5ae7a5848ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316150.53/warc/66b3c35af1802cc41445cbd7df5b06b377efab6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316150.53/warc/b219439414b2b5a84013aba31a822396ccd047a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316194.18/warc/086d8ddf376bfbf51a3aa25587cc968d033f977e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316194.18/warc/2b8a7f79f2d8cfa6940b1b61b3baa03c7ea89795.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316194.18/warc/3002737390be22c8ebf63fb71525d93fa39c0650.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316194.18/warc/5890a8b4c5c1e41024dcefe8d32ad77d8bbd684a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316194.18/warc/88b0b62ca777fdb44aa552512024c58c1b1b57ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316549.78/warc/1e822bad8df194e3ba39b1d21cea964fc6826e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316549.78/warc/251a06352793aace79c331d8ce75868f668baf09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316549.78/warc/5cbd4ff3eccdf1e5b4296db7be36bc9193147925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316549.78/warc/bd7ed6940a75f3a7c68c43337647215711fb34ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316549.78/warc/e99d1ef82fe3184545b926cef9d16ab6afe305e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316555.4/warc/4563075ea4e113a81cf8cc2018747272cf322e95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316555.4/warc/5d2d30e6e2a10784f65cdfcff9221657fdd2a606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316555.4/warc/680ea70d31111bcd16c06c510d0681962ef96637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316555.4/warc/6fbbbb9348081e4f39e147bd1c95b21f1dc4531f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316555.4/warc/faad202bb60e27105c0f29c71358b271ad5cd6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316718.64/warc/564634a1a8f0efc972f958cfcea621781c8fe9a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316718.64/warc/ced419594208d906766eff131b220fd205370149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316718.64/warc/e6e100ef3af0e82c39a9eecc151b1878a46c60a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316718.64/warc/f144dcf0b76188c0aa15d1d6a9f68a6b0505a1d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316783.70/warc/3146b2a987ddfe6ba805987faf543faa634ab2fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316783.70/warc/345e8140be1d477cc5d565b29285fc1cd3e2f5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316783.70/warc/8321cf39f612f5e6402ad6055629d410e71fee49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316783.70/warc/877bc92b29ca7483ad87cfcb0318c574dacc83a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316783.70/warc/eb4b150f2087c646f1d34835f33aca4a2f381790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316785.68/warc/01c372618c78488ca37fe081364a24960bd2c295.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316785.68/warc/08e1cd1123782c6d0688b25dcac6ffaeb7e02e39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316785.68/warc/163ae47bf849ce86d89639b39d5386842e39dcd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316785.68/warc/1cc47e52183a5f32850916ffb941596cdb5b2e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027316785.68/warc/64626da2a33e875ca611c6e1973e618fb1f02278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317037.24/warc/0d3585f31fe5b85fe5f3ea31b620e9f5a39ea336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317037.24/warc/6040712ce349621d425075d6cebcc3314cb03ca5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317037.24/warc/ae849c53635bee83aae4ea1326f27ddadf1f9a35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317037.24/warc/bcb35b28f28e5d0ea8587bd4c0d411585c37b849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317037.24/warc/de06d7243a2e1acbf4eee92443dba2317afc4d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317113.27/warc/27866278cf27cfea50a3906843c0ac787f823e03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317113.27/warc/37f3821a53cc538c141cee10190094a56da976c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317113.27/warc/6d06630ca63638be606ee222de9fe1cf29c6dbbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317113.27/warc/9cc65e6787587051f4dedcf0b1b8b26edbd4824e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317113.27/warc/c7e5afdbf374ce70230ecb4e91f767af56fbaa08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317130.77/warc/4dc1632432fb1210aebf2299578147947de59f40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317130.77/warc/55e7bb656d0274fa8910929a3a4d3b957b71ccbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317130.77/warc/f5e5e93ad45a126cf7b77ed514643774c29e3e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317130.77/warc/fb861c66163c919926d97655c251acf6afbe4f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317274.5/warc/0e1ea3cf823342d89b844f80af8f879760da1cde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317274.5/warc/2badcf769a85a1d55bebe47e596ffff91c72b89d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317274.5/warc/635dc2e973cdc290fcd48c1739d2b9584067dc96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317274.5/warc/6dc3e31067b540142e40e50566246667c7fe194d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317274.5/warc/741b380019dd0bfe1c4f9a28b00104b51da1df1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317339.12/warc/27c6329e0675437b2f8ce2236b717a2586f5a860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317339.12/warc/4fd01c2dd67a42ce3822df5a7c9b7f7d09d5ea86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317339.12/warc/91516c442d818f8e09d2d6a3bdba0259b3dd11fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317339.12/warc/bda71a2c3593c95684addf13d01dcce0be6adaf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317339.12/warc/d1204f3d34906f1576ac346c7c578b411845532b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317359.75/warc/2911d13c94f76a886cc0b0de2d5e7cd6f8d3c20e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317359.75/warc/4fb5672b2fa0407ef675594766a0b33b8d5189b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317359.75/warc/8833b63832e4e1720db1b75aa8b47aff58920266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317359.75/warc/d39b1908f17fd7ce9c68e26ee6dcbaf2dee58ed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317359.75/warc/ded3a9495a74c1e7682f2bca2de9d99e9b6bfb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317516.88/warc/96943f4b1b2d51d6dd5cbff2099d62fc7bd62729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317516.88/warc/af92655c7052537c7bce92f1c00629a1dcd2c4df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317516.88/warc/c9b17fd4ea3990f2ae2ab54dc2cb7d4b44d221f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317516.88/warc/d31a6397ab05b454ef91bed0c581d716fd69bca2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317516.88/warc/dc391d8e90f03f110f2fd9bf81dc2c76f5c8c492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317688.48/warc/3617748ec735d94f7bab613bd72d59a72d771edf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317688.48/warc/5cfcbdacdf1d98959c257102ccc29cf7551e523a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317688.48/warc/6fea2aacd77e8da998aa55ac7407a36187525a61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317688.48/warc/f90227741c9881b5982ab54fb939e7173f60cba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317817.76/warc/2c6ab8c70a03af31bf5ed6353edf4d7fa68fa2ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317817.76/warc/352ff98291a05671aaa28914c22c406c16861e2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317817.76/warc/641018262ef040198ce76f3de0e275a3e263e5e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317817.76/warc/741068685650931bde984522b90a40fba11629d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317817.76/warc/e61a555fb7074063aff4b9935526a84180e8a331.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317847.79/warc/05b661f98d5da23fe96fb7acd50b1e40ce387ad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317847.79/warc/3eea89d5b6af25e7ba58043c3ad4ebb4b9268ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317847.79/warc/4b869a16557196d38322162d238c3b4d59f46056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317847.79/warc/a3ea739a7457c6f05757c7d40c080594e2ac3ce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027317847.79/warc/f1a4e41f1fb69c814a544c9550bba5c9fabdab8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318011.89/warc/3af8a4f60fd120d28469005ca542f7522eddd23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318011.89/warc/74fc0eb5fce37b1fc24ce6eebea9d5b462a34ff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318011.89/warc/83be98b910120bcc15ae31cadfb6d868750b7a8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318011.89/warc/9ac0e1dddb49686487943d17bb35ce899048e8fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318011.89/warc/c80f0e749da0fa3a13a0354b3a8f3b8690d7d46b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318243.40/warc/0ff1d426a0ba8ffb37c55f152ba38dc5ccbbad1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318243.40/warc/3a75665fcba5550637ce6525724eae3c27936a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318243.40/warc/48c8f36b7244c6ce659142173c41c0ebe738a7f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318243.40/warc/e4b9c82873a0d22566eb7b4d71da9253be2587e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318243.40/warc/f9f4a92bd444d244b60fbecf91b3731cfbfd12d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318375.80/warc/2d81b1d78588f657906aeedfaebebe1ebbfb48ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318375.80/warc/8e6e838a748ec196dd2d0537944b04b9c3ef9dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318375.80/warc/c2db048aa011f547947f9ad2c6e0f1654ee831ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318375.80/warc/fb291315437647cf49af9366cbbb0b3f0098344b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318421.65/warc/0e31cb53738b0ac55b2b3605e7a2d75bf46150f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318421.65/warc/370268e66cd2d89408d127ada60b918d059ffeff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318421.65/warc/495dd6b422b29b934642f5fd580307e728744528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318421.65/warc/53880ad93fbdd65bfb361dbb78a4756c0c7e1baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318421.65/warc/b854a2d22fcbdad520577d3ee9a6adb8d82ef64b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318894.83/warc/131d9894ed1e4d71381b361266ac79c4a621936d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318894.83/warc/35b388387f8bd38fc847167ea6cc9021eef1423a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318894.83/warc/381e8aeb2a27eb8792c2d575db414bc057d05004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318894.83/warc/bbce6fb5312001cb270bc1bbc60e19eb2e939421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318894.83/warc/c50aced1961b43e7b26e446775572b2a586ce88c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318952.90/warc/049aaf2ff5eb50a7868b7ec97ba0c90eeb82b84d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318952.90/warc/06ffcd898ba504a4569ae065e3eca60cbfec2abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318952.90/warc/0a7d9f8fc52259a96f715c5b5a1cc8f520fecb4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318952.90/warc/9151ba0fc81a1f9ce741d75203bc8081579792a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318952.90/warc/c2156c7b78ad59d0f3c2e85ac83dbb46351a2fca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318986.84/warc/1c4b4be7d740d51527abf90f5ad856ce65b064fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318986.84/warc/1cbdba815c5e3184d7d00e1fd56535cd133d584b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318986.84/warc/3082d4347322ad56a8715ee8df5e538179f083b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318986.84/warc/99a891cddd18d55f550260c15148fab50c878e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027318986.84/warc/ce1cf6cc5ce3d0a353142ac44739c42361372c22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319082.81/warc/7952157a686d4d0caf8341b734e778cc0108a3c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319082.81/warc/9f02252eb08662860dda6e71c2084fdc713a3c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319082.81/warc/a522d4469a21c23586020331d762ff60bf3e0c23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319082.81/warc/edca4e620b15940517b6945da92f81a7708349ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319155.91/warc/102366991d39133014fb4d25b2d1933cef7f3da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319155.91/warc/61e4f3b4bd82efc4bfaa2d7a70dd1c508fd91a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319155.91/warc/97c657e95659439dc58e05e9890fb5afab2d0619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319155.91/warc/f4dbe8d85487c21cb351d839db3c4c47188b3e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319155.91/warc/fab12128be1144bf34b08f30b550b3e14dd2a601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319470.94/warc/6e923f3efd804f987ee298ad3213280f2a103bad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319470.94/warc/85316aacd6244b87f34dce97246891f5ea524736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319470.94/warc/ae9fc2d06add91609910f5fad00ab3adaa2ed829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319470.94/warc/ccd8fc5a2fa604425fdad36b6abbfc1fd3dd6bc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319470.94/warc/db6e1e58022e62f0573bb5063fcb7b42dbb79534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319724.97/warc/18088209958034fcf342bf6d0ebab5fd2478ee8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319724.97/warc/4da75cecb1b9897b5283ad8c0a334ee0f86e4d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319724.97/warc/8e373d34e1d32d66cc0dffa8df41864d9cfc556d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319724.97/warc/90b145eeb9d9354774041bf4920233991e68db4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319724.97/warc/fcfe2c5058a021d3a0f7bccee7efb15462cd64c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319915.98/warc/1bdcd003a08a0dc21e510074d18f6db8f595f8f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319915.98/warc/20321035e56bfea399f53c25adb34319df810b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319915.98/warc/66a9d7a5542006917a0ad866e474083c824d9aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319915.98/warc/7fb1ee6c39ebea71e76d6e5a72bc1658f93af177.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027319915.98/warc/d8f2d177b16e13ce3630154c5c68a948b0336779.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320156.86/warc/2c41746447a7af69cf8460746b8991b0d16ff53a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320156.86/warc/4d6c0d3b81adddcd405538acc06a67395b9452d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320156.86/warc/84bd42cbf94a62332f09ca3d33b77cc2bd36c08e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320156.86/warc/f278d5680e5d0a10b2a316e03729c6c2fa0c61cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320734.85/warc/2e0a318a27bc5a5daf51d4c7bc7add5f592bab21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320734.85/warc/3c9291988b2df515c244c4bc764bb70a4afe98a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320734.85/warc/992d643f54e5a2cca8041ea856fb1afa381c0db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320734.85/warc/9db5dc966af34b215f7e46847c3b2c67018499b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027320734.85/warc/a818b8886b33220e3bfce79d3cdd08a70e7069be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321140.82/warc/499f91ef6fcb637258dac75587c0831028430452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321140.82/warc/a4e98db3f28edb3e1baa3358e4dff5bbb230d190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321140.82/warc/d41ec9641b3e740c02e3e11220db2d42bd1449a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321140.82/warc/f849bd7aa63f151ca5f61437397364531e31d401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321140.82/warc/ff1cccaf8249d65e1cb3dec4a26324b578e3bff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321160.93/warc/4ed88ca095819ca4c4c9b121aa7eaf5e099e9db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321160.93/warc/94a52d667770c8fa2a20b2006dcbac1bbe6fb914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321160.93/warc/991d7c4b0d9f3198dec94fe7ec41fc5e28cdcb3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321160.93/warc/df74761fc9d4920a33f288936be2cf5900ae1c37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321160.93/warc/ffbb90c613c49811f024e18dc74d989c6d00542a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321351.87/warc/1d04af0ae772984f0856ea5fe9fd48e074133aaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321351.87/warc/3edf3cd4714f0102e6fb54e33d6c56e30029c7f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321351.87/warc/7aac1a27bd4c5fdca67db0cd38737805f5974400.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321351.87/warc/8bb6f478fb15f9266115a8caf61c3e7219c9e444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321351.87/warc/979276c1b9df7b6d28b51fd7ffb7af4423d159c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321696.96/warc/13174f655104fecd7878980618b359a90f6b3044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321696.96/warc/30b94a9b1d750deb9f4ed65a253e226186efa881.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321696.96/warc/3a3b9d0ccbd933fb6e0e2dcc4ad59796da883ea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321696.96/warc/785f6af2f3207e6ca2c564bbc4372f7f1c888312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321786.95/warc/940ee35df88d82b8129a5f10f12d516a0825da84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321786.95/warc/d8e5c89573ff2416c088f32c475ddc022418670f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321786.95/warc/edb5891ad1dc8855cbaf39ad9bc509f9a40b610f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321786.95/warc/f5fe5b99a0a16bca742bbf7a10f2a258de9fe433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027321786.95/warc/f622902bc09c2f3f80d0e227509f0302e302a6fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322160.92/warc/085d7a0dd3d528a9be2f152752f909cdf63252da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322160.92/warc/9d94e1527609c79516c62fa5100ae83b32c30707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322160.92/warc/f4b3def8b67377441d277d4aec2496f9668b3e1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322160.92/warc/f5d2264e83dd7e4e02f599813a6a851d5ea18188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322160.92/warc/f8f0f49ddfdead94d468f39cb9a0302436d7ca2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322170.99/warc/2bd3d74be630053f0f5554cbee9cc5d3bdbd93a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322170.99/warc/2be7d9b7be3ca9c2fff039772c4265df6df7699b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322170.99/warc/5f435ecd6f55dc6b6b3fe49f63149caa96e6e02c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322170.99/warc/8eaa1b9d3a05fc103a582b42898874461ddd6da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027322170.99/warc/e6fd6324a30fa4a08154c4b5ecf24394ac9418e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323067.50/warc/1811c84c1be7d97cefe56e2299e777087937c0b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323067.50/warc/4d001b1ef26421123ff6258eab4a4c60662ae6c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323067.50/warc/855c31d9a7e31535710632b938ced46f2d0d76d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323067.50/warc/9e08b71be4e48843f6c53531359b0067bbaf2f7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323067.50/warc/fa6ee2ecc0576bae13b86d7a3112c8e466d07efd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323221.23/warc/142f186e2bcbe5d4029ba5aaee6361ffa9de8c32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323221.23/warc/1bed345097bccbb538ce19753e1a22424a9e9ec2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323221.23/warc/86019dc53ce3cde91d225622b174171958704860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323221.23/warc/f56150415a4029ba5d3c1a7493de21b858f9d1b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323246.35/warc/027537772b8de30c9f4b70e0438e21299d8a1512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323246.35/warc/1fdacca90ca18c57cb471d417c72907940851cb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323246.35/warc/215d8c3ebb66b99742840d284af900f0ad03a9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323246.35/warc/9be51f9fd83f2e922396bf9d46885c45c3f42c07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323246.35/warc/ed02d959a131bc2feddbb1556e3c31b14be30f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323328.16/warc/703de4c8a86e3035cb8596099b453447eb05e5ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323328.16/warc/bccc5581915056d1d46a428b62eab34b44e85994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323328.16/warc/bd69b2ecaf5ca6ecd48988febef27461748aaee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323328.16/warc/cfc6c37fe690f28ae8dcb567f469e4ca2831411a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027323328.16/warc/ecda883c54b5376b53205e2f37d947d44454b4a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330233.1/warc/1c3dd7590c77bb60b769d4c3d796f00c8b670fa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330233.1/warc/37486216c26613e8db72e2b3f68500a73d747f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330233.1/warc/835c8b3446a1f4474792b02ea886c9b8550866da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330233.1/warc/dd662967662e5db524f37109c8d8404190cd060e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330233.1/warc/f10125b59f140eebe680d994d47178846d81bd4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330750.45/warc/0cfc1bb8e8dac6a4deb67d38cc6f20eeb5464e0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330750.45/warc/3fff2c3e051fb99b2a20aef51303c39a4b0e5b66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330750.45/warc/b672bfc8060add303de4f027860c8adcf8117f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330750.45/warc/ba4643bdf4392e03b9863f794af76ba8506f8bdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330750.45/warc/cddf188e1f5f3d1fa8be4166f3393cdab3756080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330786.8/warc/1f8f7b3a4fad0d1c8b8ea31597ff93dccacd4397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330786.8/warc/b17cca2d931855f7ddbe8f7e3695bec353db00a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330786.8/warc/b39ad8c8627bf911212ddc216c9cfb0c00aff272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330786.8/warc/d802806a6a21e9a1290040f93a9d1bf23f76012f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330800.17/warc/20af27e75be66e13e48319da4c801c08b3bd028b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330800.17/warc/2fd572ab48034622a36f0da7a6a1c65665b67992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330800.17/warc/5a86c3e6c8c60abd44759327774cce8e8cfa1d95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330800.17/warc/e82d7cddc972123b2289e14a7016d919e9ca35ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330800.17/warc/f31b41d12b3be8bc6ddc3b0e42c34d9f519f355c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330907.46/warc/3c5ae2690bec3551012c14fbfb0766191eaeb211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330907.46/warc/6ca2b4d724fae0049025639c352d8dd3bc83ead2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330907.46/warc/911b8c740a741c6b0772124e5477177d92b95e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330907.46/warc/915a616311996d064a66f209588a9e0db2d096a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330907.46/warc/a0cb6d1c5c4b0a818eef52efd93f98a8f4aec2b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330913.72/warc/2431e888e72ed35ac8dbf8417b591e99b865628f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330913.72/warc/320249a11b38d23d0767f74124e9874563fa1267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330913.72/warc/35358c0d8a0758b0d6996e653c267adc3e265ec1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330913.72/warc/4eb2f3e558cf7e808b64bc218af91ba78742b3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330913.72/warc/60e67733dded1b0683f03270204f001f16b8a2e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330962.67/warc/02c3d649f7f2f66dab997f44dbbc30f2fb34b0de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330962.67/warc/22fbdd5866738ae507de092e4ea6a41c7cdf98a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330962.67/warc/29d7e7596d4ede0423dc8120113c2d1a18502e67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330962.67/warc/7f37ce85d3407712452be3e1f2b37f95d4dc34cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330962.67/warc/8031d6c37d2e941f70120d1e8ddfdb70b2790ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330968.54/warc/514f2a4b7e21e73197c8193b78ff9504ec12c086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330968.54/warc/74ae7b031a673460aeb1682390b52a312095050b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330968.54/warc/82e254569ae07f83df4e977c2a1acd9811a55a91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027330968.54/warc/a4813198fe39eca1440ab75a7ff6633b840d85c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331228.13/warc/16882341a5101bdb7a451952649a647a54b6c993.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331228.13/warc/29bfd21e96b8351408702f543db49b59c2760661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331228.13/warc/60c7f1b545bd75f7141e0a10af1d82973cb3ef40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331228.13/warc/d804d3563ddd423f0b11610a24d392babe811e0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331228.13/warc/e57110b97a367d025e06a8549f739123555d5cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331485.43/warc/0acd63472662c5190b673669133988bb6dc6a617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331485.43/warc/125fdf3697278b2ba506bbc6f46f65af42056f4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331485.43/warc/31986d5db9db738e6473821c804343917d5d0413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331485.43/warc/791e0d2634a90a04b2a4cf4750b79e8a7984fc84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/1566027331485.43/warc/de165864fb4fec9b7c9352dd096b299bdf7b5932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/15b08b419b9e17e384d534013978af4249422a20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/17187770333c668f5791bb05d6dbe0f9abf53bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/21fc09d4ea33d658fb635b0f788bc6dd7ca2093c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/24b4412a3b243f287a2dce9ba4531c680f645bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/25435c748832fa8a8df2fe324f838189355105ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/296c9e11d68c06724f27172a23067bdde4663a5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/29cf3855e5ddee41fe614d5dfd89689d13c50e34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/2ce2e82b2ba7ea1cc99a10548a81a7aaee572b90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/2f3bfc4c19e53f3ae1ce46778dde52f569daabde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/2f719419cc68d1515d2a1386430341656aaf398f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/2f81a655fe2c46b23e3dfc7d49583ca19f703e15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/3af68df8fc33fd71f2706f0b3223f38358386370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/3d12273d28f9248e2852fd735b4e0cff1a3ae7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/41a4daebcac4594ea077625a2fb25dd837ac2730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/4410b88bf5ea17568cb1ed35132687253d12aed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/497e6d5df0fda3b4ed2eb642057596c049dd7fcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/5115a8ab2a5882b55137bf9e5bc73e5f3d5cc537.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/518f5f96e92d61f4a1328ad213a4b9440af3c871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/55741fdb8a4acbe95c6581ae8b1dfc4a8be4ba18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/5d4d966efdbe6c36b68cab4d0f9439436b9a9573.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/5dbb732a8556c48208ff2580ffe469bec8dd4878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/5e5431734f5755819924864de28f605ae45776e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/5ec6420bea9d54efba530ee1978038bfbfee5036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/61f6d49a57724b2229508d4d207a9abc007bd669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/689298a9869eb87f1037da9f418e3adb85dfbcda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/6e7de21eb6d090ec3f60b22976404af45ee69dca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/72fa169bb72d35952571b008850ff9b00cae9276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/75afcc5b8d45abf80ba7b954386ad8d44cb80b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/775436aa0b9af0109fd4f9fd63768094509598d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/7845f102b59995f61f7acc51e79a2436aa2e4cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/7f52073220841bf0510bbccc1a15bb4f0134c0aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/82876a46f6b6581b8789ddc3b2f3584be267bdc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/87b91bb8f756eb2c3e7a8d7370999f9a7fb07f9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/89196716db104a278ed0280855443e903692862f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/8b61bce90e37ed146521b9d5e7ab6938a1acf28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/91065bfb707d1e364fd8fcd04db05aa9bbc8381e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/927a714f84345d0cd6cf045f2d26c4441a2debc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/96d7a213913a5ff886a7bc54df78e3ac86cdf168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/98122618c80d5dc7e89c732d8c4cb31d689eba6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/9a2c90a2096f29961dc7b6183729b581f0ca3bd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/9a50069e3c82bcd8fb60fd540e02f886b5e7a7f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/9f01996527bd642bcf2ffce255d6a05d358c0db5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/a23fe0f3fdbc2279e8669465e84a68c02b34e63b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/a3ab114b5d071b78b6cd649f1a26114002c62477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/a8552523c72bcfe21c39024bdf805fdaf1bda147.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/acc48b0faf53188b891e3ade4dd833622fd6765c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/accf60cb4e2bd8a0e35438bf3e46a1685f631ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/af0864f80f14f317b27f5639413d7fd1cd55d5df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/c0b9fb4f844f85e7e86f2347aa0308b188cbb4c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/c23b4da40bbe239923661a6a7b75878e28fd518b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/c41a69d9b3c789224c836abd90dbafad742b41db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/c743cf9b33fbbc6133b22af8057c92cc0ae20058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/c9bb8b839bd0474cce4dedaafc7ba5fff8d237c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/cb61e4467e700843a740e4b23df09ca178c906dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/cc8984e483bf87240ce603f65f0671b17dda7bd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d10b453df76477c999095711ec057413193a5f52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d39ef3f19a881ab7bf58ab99cd0236f72fdba40e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d718ba8083c5df4ed547f65c3ee638ca72e80aa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d758f93c1aceb4442b47a728eed0a170bccf5d85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d7a5bed24019963303dad65205b6a89b97711d75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d7c1f2f689012bf375d4e76378582b7e0a91bddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/d96eae3390dc73d20380c0cb2259a4766e9c24fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/dcea1b22ba9d31e9ba7e01ec5ebcd950047289f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/dfadde20f40b638f97eea8a95019b0d0b283c15f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/e4ba5673f27b527e9e98b85b592b4010a00435ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/e6e5174aebcbe062e4f08a5070eac7e98ae4ddfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/f47a817d677ea3f0b344aad9cc7c6fc1ead993c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/f691b0cf47d812e76a66eb3cefb30b959d98f866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/f7b1b3e26fb52c01b3e15ce44709f9d53690a7bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/fc58a004f4eb4c370c40266f9593f012998694e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/fea93a274ddcd6a1f4cbc5d420e0f9750b3aeb97.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-35 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2019-35/mix/input" + output: "/tmp/cccc-CC-MAIN-2019-35/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2019-39.yaml b/configs/cccc/mixer/CC-MAIN-2019-39.yaml new file mode 100644 index 00000000..f7194919 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2019-39.yaml @@ -0,0 +1,674 @@ + +streams: + - name: cccc-CC-MAIN-2019-39 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/014f726a7e27777a552c3374a3b29f217f5655a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/0871195aa4c67dcbe23dc1f08981fe9404002181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/0aeac422bc310a9122417ece09960b29b714197f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/0dbd61cdf0a71af738f344c56aef725075709389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/13dcaa0de8c94c44218d6c5ee4ec347251e89f5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570740.10/warc/34a795f5c17efffc9750f48b3dd412479adbfbba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570740.10/warc/6a64e2b55299ef27595e357d05fd1d405bb2fb40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570740.10/warc/875fef2ab932ac72911f22c035343f5fa67d1031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570740.10/warc/8d0eeb0f82386f529f9af481eaf337b7c188e2de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570740.10/warc/c235e5bbae6170a973ca381a9311ac9f226627a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570830.42/warc/1d8d908593f80b5383ce231f390feadde47e6de0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570830.42/warc/1f1263ad90cba694a689c7b1b337dc70c6d6d457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570830.42/warc/715c99bfcd76540cce18c63576cf618ef673554c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570830.42/warc/74ce4141dd542e379c54f40d90c180fa149dfdcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514570830.42/warc/990b792deefd53ddc26f56fcb770d83887a1a506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571027.62/warc/27fc402d6a9fc7f113565012f656dfc8b64cc0ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571027.62/warc/714fb813e6f6b350b58f9541cc270c5584cddc46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571027.62/warc/bb058d8a2f1bb91f0a4e1c9dbff54614927eaa81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571027.62/warc/da7e379b60f14e3c592fa111e537899a34ba4871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571360.41/warc/24ba2d74458273842b0df4a4971f9daf6af28af5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571360.41/warc/540b40df972b962b9170b5044bfcd047c47b8630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571360.41/warc/7fcbc3c53595900a7fb461cbf8cb34a1e76edb8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571360.41/warc/aa72bae0fdb692c3eaed55b03e98e1211db4b5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571360.41/warc/c4f568cfd3249126da72e8db2c93a46bbe188b4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571506.61/warc/20cd590b3430da20b0d1bdc36cbfb3bfc42b1364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571506.61/warc/27a0c91e82fa0b5896338b434a43ffb4c1aa4c83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571506.61/warc/666d1d2f8cfcb25ccdd29d021b1a1ea3a9ae811f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571506.61/warc/bd7ce26b7b6e7df8cff03d33c5e8915f832f4f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571506.61/warc/f04de30e53045c349207148fd606b38ad0588e18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571651.9/warc/1b30616c62c94a7a54a6d7724fbffe82922d4f02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571651.9/warc/2fd530510828b9ee395a75799851529378894d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571651.9/warc/3a87ee9e57f8c44726768bc8402648d71e0f6e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571651.9/warc/b3375ab8de13b7185215d2ee400d7e680bf4e839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514571651.9/warc/fd1ed62bb34765d26155fb9a4c791df950acdca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572235.63/warc/186b0f8fa565ca707b20cf60b9e81f70e9476f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572235.63/warc/72e50f9a16e61c318b778c914d0e0993f9f7b551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572235.63/warc/9c6c5e6582d1608c7ff6156bb154c339bbc87e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572235.63/warc/b10cc4b3e1d5013d8269441e2eb8d811c064d7ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572235.63/warc/d755a9dcf85a98ecaaff691ec2100c3d50867f1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572289.5/warc/11bf943ebed1062b64f513f9652e9c563e644722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572289.5/warc/5a819f8d038b79693893a405abc4764a606e0000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572289.5/warc/6fc930cd8157ca413dfe00744db15fd5009927eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572289.5/warc/de9be143ffbaaccc5fc47cc3903aec9327e9e82c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572436.52/warc/2ca5b72f5bf39b69cabb413339bfba312981756c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572436.52/warc/4082b197f5fe012bf162a0cbc5b3db88d05c3c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572436.52/warc/4b8c8ec4e0d86d57cc31d9bf8d4bc86bcf94862f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572436.52/warc/d4936692ef76df46bceab11d1197c0fea2a0d3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572436.52/warc/eed9a9b4500e6fc77008e738b2ff84666b6b408c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572439.21/warc/11e359d9d7250fc7a3dc50d1cba0693ebe67e1fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572439.21/warc/25a01ca4f35945ec2349dd60bcc5632b0b41f0e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572439.21/warc/30149c261611a3b15295cd83b5b7263de112c324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572439.21/warc/5c69f08c68674331616e616bc4e54419e8710e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572439.21/warc/e6da87b66c7b264a0f9a909861ae76d6bc3bc0f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572471.35/warc/3c85d094bfa0241ebada6f0b491861429fd288a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572471.35/warc/67b257e6085134995c6228333827f7239d512dfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572471.35/warc/91d1e98b793648f360f75860659e1a74754f687a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572471.35/warc/ad256cc433cec62feafa3249eb6dc50be1d75646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572471.35/warc/f856e7d95e59882f34c25941242868546632e957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572484.20/warc/180cfb0fe708963da6e342817f65d5de2050c591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572484.20/warc/3cdc13279a7b5c1eb8d4184dcf7a20a32b361f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572484.20/warc/64167997fbc2eea2b529ee8820607d67ecb5465e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572484.20/warc/9fa97fc970c1f19e041211d84203dcdf8b1a97e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572484.20/warc/ba02141726b4f6659be97abef63c3f9165f22877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572491.38/warc/2075327269ba396f6a412bb5a650ae14d50b3298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572491.38/warc/3b5138ac111723aa4fb3e126fb9d61fa3775a265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572491.38/warc/7ee3f61f72e8ca59413393f4e491c740a3bd6ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572491.38/warc/f6fd26c4c4b2aa2c4b5432635c436d501514df47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572516.46/warc/3d46993a8a94d798603041e3f527483a8d085a73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572516.46/warc/72351e133751553989eac6d7bb318e96bee7ae49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572516.46/warc/9466ca7094105a6a8d900312011ab84fb2265af1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572516.46/warc/c199df685d688738647d2907550e71fa5785bf3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572516.46/warc/f7c1cea85b7c90f4cb890109eab747b06c5d951d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572517.50/warc/07bad11e9f27bfdf57b344dace088aa5bf590768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572517.50/warc/43e9fa5809dbb05158566cd274b1ac951c01db16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572517.50/warc/49c538e654767410c4da7f2f3b3df297ed9e1866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572517.50/warc/7fae1a3f85f218dc5cd1b8677f606c425f67a18e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572517.50/warc/a714776bc94cf88c50b41a6eb76d43c344706a39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572556.54/warc/308b45f483c3b50edd4dbe3594f2efddd1784724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572556.54/warc/b4ebf6f8e5e20d93caaf35637b734657d0e0fec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572556.54/warc/ca09c3660f66c547955750de915300d0061c8d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572556.54/warc/df72e2a17bc589fa750b8c44b30a9d92e5bcae92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572556.54/warc/f312e32b5c2ac875f18dfbd21b72a8d7cd2104af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572744.7/warc/21725b009c9667947eabcd7a10f9bd1af540760e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572744.7/warc/2e6486ab966da7af62a3af4a0b6e8dc96e2237d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572744.7/warc/6fb8a52e99d5ff04526591131b10abea6291fb9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572744.7/warc/b1b89b51e57c39797bc039984037fb2cc105709f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572744.7/warc/bec4cabc4337cf8ab17bd6992970452dcc5a6d89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572879.28/warc/35434d4eda2c6835badd581dcdf87a04203bb55b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572879.28/warc/57909cb3f9c952d61ff1db4a83911502d0abbf05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572879.28/warc/859d33c31ed984e4560f3e9dde5a00f884bec0f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572879.28/warc/a6e71c5d51e3156c30d10ba50b5a24530c8bb6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572896.15/warc/1640371c3aa73b172d06bb8a26fcf3aef29fda04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572896.15/warc/4b81beab8074f2679b51d24bad2550a2ea64f69d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572896.15/warc/809d432b694012a04158afd2e7706a2fbf643821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572896.15/warc/8ae49e14b0d18a422561de003ba4ffedc630b169.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572896.15/warc/c26eaabc708ca7f2fb4ac89b7a1617c7dab0a686.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572934.73/warc/1ae9b09c2086689b010abe3faf579429ecf70589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572934.73/warc/5479d2bf1cbeaffe8437a8099b59e191792099d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572934.73/warc/97469f914e00755bd1b94f9a13da0039c1fbfc9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572934.73/warc/b242e4dc96100972275a6594001c1d7d44f0b701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572934.73/warc/c85b239986c83cfa68aadfe0fadb830d7f8308c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572964.47/warc/006dd676f5823c4e769fc6055f586fcdfd482b75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572964.47/warc/0541e7ca03e80e5efef75e23493ea7fe7085eb29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572964.47/warc/301608535d3b37b6a3a8907b08c99fa6514c0152.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572964.47/warc/87e0c64334d141979944b131a483d280a49c1e98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572964.47/warc/d8e078a10895938afd449fd6317cdab04eebbf26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572980.56/warc/0477d01ca8166782c4e642b66a7f20b5c84c025a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572980.56/warc/68da8592304865db3948daafc56f63712610f100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572980.56/warc/8a485f5bcb86cc26e6b3fc533294512227270e73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572980.56/warc/c017ad25bf744ef72a3eef960e4e829aa8193356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514572980.56/warc/d560cc3040312f9ce5b3f1f3b1279ca48cc449a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573011.59/warc/37497371540aa07c7c72c9c29aa0e392522f95c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573011.59/warc/cb135ae56f709764881aaa9a6259eb6f4746d1c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573011.59/warc/d8ea38d473fdd2c69e9b8f23320bd417aaa5c66c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573011.59/warc/dca93a365de8b2e1d81104beaeb607b8b813829c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573052.26/warc/604016f8095241d436bf3e4137351bde06ffdeb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573052.26/warc/94bc77bc0a78216f1f829b6c3fd9e598f6bda7ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573052.26/warc/a8abdacf43daa559f265662eaa24832848b4d465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573052.26/warc/b3bf4ec8f5923e66bfff3bbb83eb7275a1a933c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573052.26/warc/e06ebe4374026051d88c5e1e3f469768c021d54d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573053.13/warc/11209c2090bc2df57d37d3c502a663c2c28ba9dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573053.13/warc/4360bac1558c962b7e7641fb212a33a24f3dc701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573053.13/warc/543d095bab97bca14ba40204037a962604ad8ab9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573053.13/warc/74b375f83b7b53b8965515b70ad252daaeb8871e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573053.13/warc/9b9117036b435cc724cf892a97e10493c7994e9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573065.17/warc/1c4e194b89b2f5613a297c182e19d6992e7533ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573065.17/warc/2b5851573456f5bc1186edb31b2fd496c9b72987.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573065.17/warc/3c4f4734032482a81632d7875bfeb2f250cd40bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573065.17/warc/7df89c938935a66cc99790df1cd323c90909d2ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573065.17/warc/a86042532e86b128efc3b409023ff7baa4369792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573070.36/warc/b5f647161c9b775189e9486ae53611324c849269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573070.36/warc/c5b30c3c44e52e99f3dbf30632b304e08988915c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573070.36/warc/c95f4a1e9172114db05d79c4d07d2ab32c50bb83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573070.36/warc/ecbcfe92924eef121fdbf19f5204cd43918e1a8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573070.36/warc/f2d49f831e03e570fe2101127b10e23e12e8a39f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573071.65/warc/630d388cb934f6a6577367cc01b6daa4ab006a5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573071.65/warc/782b0862d26e2df872ffe88efe9ac54cae9a6366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573071.65/warc/9295646fd30d63b56a21a0c5e6511d14f122d65a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573071.65/warc/fdbabca1340ce6fc245cd0a0e199d545932ac3b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573080.8/warc/1b5091951d46f31b78e3005adb0015b9c7ea550e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573080.8/warc/275a91f91cc38caafac979995a2dc3e32ed91850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573080.8/warc/35624534eb8a60101abece434d06668b4223fb30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573080.8/warc/46da7f7ac33c2c63f3d60f1a03a7698bf56f56bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573080.8/warc/770c2878fcf38ecf47e6bfa97d275487e00dcdd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573098.0/warc/4f448c85554916aaccbc4a842bc7ac653e7e16b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573098.0/warc/8ab4fab0b324d203372783fa3a6f4a3f6768c3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573098.0/warc/8fe6c8e7916d1efa138700e6ac869f5f2d56d98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573098.0/warc/a9d92c78b14e4730171b02fb13f4ec0483a5689d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573098.0/warc/fa4d253240b61868e7716b2dd332a7bd161f6198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573105.1/warc/53014e0261b5617a3bff5e95191e67fb08ec343e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573105.1/warc/6c2cad07a10772ba79a8adbce46b6b919c2c28a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573105.1/warc/7d398fadff0762b8eb98ffbedf466986ffdd6d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573105.1/warc/af1dfb8a84511b511ba9914833a0b369d9af281f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573105.1/warc/d9f7145a99a6b9c44256a558ae549b16e730267d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573121.4/warc/42002b44627425f4a5fb06142ba3ad8683b6bd15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573121.4/warc/46a3d4d39a19d31c77a4207ced19bec35979b7b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573121.4/warc/91f6d942e5f0f5f3a1010ef7dba7cb0b6fa21bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573121.4/warc/d9552598769defe5c2e0e7c9a5b1d190e5162b0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573121.4/warc/f3bc96712cb3b1df60e3d861638879bb3480dabf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573124.40/warc/1034a67f23a8483eb92ccee3ea41bba033d1dc21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573124.40/warc/6d8cfe9d7bec407a6abf750ac9b0961b66f3aa9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573124.40/warc/9868d15a3f34fc2df5f75b6f04efb520f5e7a2c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573124.40/warc/dccd63d9e038f0bc3534b9665de97848cc042a81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573173.68/warc/0e7dae0a1858155f4272e51c528bd1c82812472e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573173.68/warc/559d418ccfa4ffdeec99268481701cc6ebad55f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573173.68/warc/e83af7f801fa8ad5cc12c41463cd95d50e5eb4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573173.68/warc/e86b282f8eefbf53e3da4d5c22540193b3ae3035.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573173.68/warc/f9dcf73c4d703a5e25551a593dfed9b3b6534de4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573176.51/warc/1d08706d296d645cbf4fb61099634a89d4efe3dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573176.51/warc/59c786f933099d111f4feecb93358060906e03c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573176.51/warc/b15f46a173cdf51820eabaf9a4fe93d23f380854.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573176.51/warc/b340a026375e98f003f38329fce03bf3ee26e610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573176.51/warc/ea92b0ec9583d6c2773c0d353bfedd47329e8214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573184.25/warc/69286ab2c1bc9a0d23c278dcecfe9ec9df9cd256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573184.25/warc/a05ca6f0b7cbf884c387a857cdf11dfdc8eb0d7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573184.25/warc/aa77bbb1a1ba2e41d40e1ca9228e4658d836a646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573184.25/warc/f1b43cb978134b0e7683734c811aaf7a67768850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573184.25/warc/faa99ba7e68ff60e751524c20fe48ba9964dabf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573258.74/warc/0211671fcadc2df12d9a2f541b4ee6c2a91b9e8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573258.74/warc/2974f77eae2cb3912a9f2d25e12f1e0a6b1331d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573258.74/warc/88bb8ec1461f885067e8ed2183a9dd3f32f55d29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573258.74/warc/93c198c1dfb956cd46c7d837953b6bb843505ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573258.74/warc/e464cd8b15c7f3ceba323d8ee3d6133ccc3baac2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573264.27/warc/1d741d373000091d82499c50f5863dd5ccaf18f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573264.27/warc/24de8aaa90ae0304d063c8f6a57c08d2d1464007.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573264.27/warc/532dca4dc98b32cbf1467c395c875c392ec42877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573264.27/warc/88472dbbfd674870f6daceb9cc3f481612a16893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573284.48/warc/33277ffbd71625e0926369ddddf943f7c0832d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573284.48/warc/4e9e73983017b50de229ddaee60188232da94f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573284.48/warc/63b35eab7c46c1f62f3d67b9fb095bde1fe55cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573284.48/warc/6b5e0e0e5fc77650d2b0c983003c2157851933cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573284.48/warc/b2222b6f411723abda1595da535d1322891c9d0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573289.83/warc/28a7f8f33ab3e4c29d6c32e8eea50a077ec602fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573289.83/warc/31b613a2cf02b656ec09b474bbefaec519a15ee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573289.83/warc/6725c063f9dcf9a571480ae5565678fe0b523c19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573289.83/warc/923b332b41af16d1a513a1499f33d2f3f1f96a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573289.83/warc/e8f8cae8c491ca7f8300a8d699c44193994d91a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573309.22/warc/2e65447479830fa67226daa6d8e22e6a8336a051.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573309.22/warc/53c74f880b0f7ea7e06276133fb9be46af4b70bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573309.22/warc/a9d320e0a45b997c182ad9c8af1d845e6a114505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573309.22/warc/dbdc234cf46d819c632296edb8773686a7e18e34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573309.22/warc/e3f1ff531132e3485620e027445f6d12ca85d147.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573323.60/warc/7dcf006fb5f5b4e23ace98a886b9c393aa8feee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573323.60/warc/a1662ea6bfd14b21783b0d25051bd6c9449d0375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573323.60/warc/d110a156660bdabd1fe75f6c2f7b6a5021d18154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573323.60/warc/d1809db0664eb56202d34eb0419c804733321b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573323.60/warc/e163028c233990af918580476998d080c3030327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573331.86/warc/096afde3fc5ad399e4800f47b9df819a0369725b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573331.86/warc/375900c13aa473e579f46ee0bb43077336490e3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573331.86/warc/9cd2722f1d419875aefdc3b2ca2bcafa661c54e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573331.86/warc/c46f1454f5a7877369ec9e2ef185e4d43dd6e13f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573368.43/warc/2341966aa4999b160088087f59269d2be32e7009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573368.43/warc/583694352b31c8ccd5a16d8ae30ca56826f8371a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573368.43/warc/5e4bd3fbdf21b35f79211a6c109e109ca35267f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573368.43/warc/dc62ecb5e9ee38641fec7756d9d124cfaf1ba727.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573368.43/warc/fa81b0df518fd2f6fee10a87a2bd1abfc7b8390b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573385.29/warc/36b7189d8687c342efe1f50d99b254fded970fb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573385.29/warc/36dc9b9de16f70f8a28c78fe0907355bdc47eab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573385.29/warc/7ffbcae5fde1efe7c59b179a427b00b0507e9984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573385.29/warc/96b76700a09c059fda85356132bbbd787e6193ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573385.29/warc/cdb7b8cd28a34b9e8ceee6530de09c671fe238cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573415.58/warc/8122edb2f710f06e1238249b0a431b04f30078dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573415.58/warc/8e1beecfa40c0bc57409bb1f006487a48f118cf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573415.58/warc/cf20934fe2aaf7cc0b0eb9a87463e4ceab859c6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573415.58/warc/d8280afab6bc59c63d8f995140ed9424b8aa2863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573415.58/warc/feec697dcf37eef90ed843c2d97f6241dc1206cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573439.64/warc/1e3d8ac390a015e99d1b6515b83bcc28dde0b139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573439.64/warc/4a38afe9cae8f9db10f8f9e3af20662d99c0beaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573439.64/warc/589172243df3607497fee81b36ab3fe6667902f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573439.64/warc/7bb51ad002aec452413e23774cb91190e729cae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573439.64/warc/f42b1213787464fb3dc7632f1e0cbf1bf7de1182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573444.87/warc/7406b3b812a4966186566b6f049030c4897b99a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573444.87/warc/768b731a9a661da18aa6823cd5724ccb99e3135c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573444.87/warc/dee78e8a4c949d28091cffef777de0dd60dfeb1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573444.87/warc/f8f82900e3d19b01be6a774d8c2a8fddccbe4849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573465.18/warc/1310c6953ee46bf900b1de3d46069345421903eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573465.18/warc/233ad59d75789b8380e56e93b4b5bf79f12c570b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573465.18/warc/96cee1948ac8eb0ccefe1a5137579f429b3accdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573465.18/warc/97450269b2b33dfe60e0f356f32bd5dd3b95047f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573465.18/warc/d6d8edc5a56c2a59148c4bbf7a7c9b4208b389b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573476.67/warc/2992c8e5964c7d21f8e96eceaa638e8a0f3ac368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573476.67/warc/5734a44ece111ae75d7a3b176bf7a8dcac578ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573476.67/warc/9027bd356d97b01bb34d16aa2137d1af98b436cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573476.67/warc/995c3f6b5aae252e49040eac5e301aac7cb14d6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573476.67/warc/e6370e7cbab35df740fb9d0ff267e1b271eeb107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573519.72/warc/7a570430392edd90e7fe61f103c026718bf8c5cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573519.72/warc/87c43b7d268058afb7da71913a78159d67a20ece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573519.72/warc/db019f8927a15e985f6e2ab6123e442262a52117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573519.72/warc/e940d144b4164faa5b73dcc43db165e7f1722b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573519.72/warc/fb4fc1270032b72d0fdcbd6cfb24eb50cddcef94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573533.49/warc/0d4946937ecfad788af5198d60d58d7fb2631d41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573533.49/warc/797ab92f24b79ad070a9fafb775c39a7e5a111bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573533.49/warc/b2324eafb773bb542b8e51f3327586a346f2c57b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573533.49/warc/be82b26a9368ef2d0c2911955f0c7687896a0c5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573533.49/warc/e1b05c41576c2d6df2da7956d808da940ce2be7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573561.45/warc/174126df2ee89871dd0b309eb50b781749b980ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573561.45/warc/2d31100769ca216c9f2a2e2afec0f48b7a905157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573561.45/warc/38d041945f4de5b5fff37751677022eacfc4660e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573561.45/warc/db083321ad1fe6c7701a8bae28a3c3d930953dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573570.6/warc/239865ca27fc5dbbfb485533690138d74c725f71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573570.6/warc/724c202261a037d3111de440b09d2ad14a40aa55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573570.6/warc/7ad0fe29518bd4c4e0c4ddf665520a589ed057dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573570.6/warc/9069bafdc99cb0584c22d2705d5a87d3562592e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573570.6/warc/d6106d10b37053be657838d3a6f7646c31f25550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573735.34/warc/02fb0c42efe5fa407072ca32d0e0c27da779ba44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573735.34/warc/33f1306a14bf07204b63d1f592d0fa8fbf2efe19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573735.34/warc/73045005142585883cfdac34c73707d0c01346f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573735.34/warc/e84e58a150acbfe378194f8fa7cccd23bc1c52d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573735.34/warc/f9003313ec8662504fc88fbce16631f2519fe49d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573759.32/warc/1cb8e251d6a487ec8d1246bf0dd1eba1f17fa416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573759.32/warc/242bc6bd0eb8585f69f7c7b8629651dd8edec9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573759.32/warc/81674d634f3965ad28a787d0e2a1d61b53ac69e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573759.32/warc/c59e129979ba9dc2a5013b0165153de401f652c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573759.32/warc/ee0554d6983d65e2ff6a584e301a08a0754c50f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573801.14/warc/34800a55ff1bb3b27d0df95b9f309fba2e26f49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573801.14/warc/6b3227b24d4c963b0c9f70aa8e94e1db950e3f5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573801.14/warc/73aa29b9bf7064d072fe89c87bb72b3cc633baff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573801.14/warc/8031a6d377ce83b79de8645049f2676ea409ff98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573801.14/warc/df2fb74631b3c03429cd74e34f56b4ae64c6a089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573827.2/warc/35591068bdf38ef3a6112c3a32e8cc047ff7cf9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573827.2/warc/4cb1b50243103e720ba009bf70d415216d9c6379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573827.2/warc/81cc815a75b3ca5103df0e95f5326d6ff3f14cfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573827.2/warc/86909cd3165088320c080831c103e17955893cf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573832.23/warc/3829550e764bb9bfe753b628d370ef359241933e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573832.23/warc/51079586bd776e213aae113771e40dd703a08060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573832.23/warc/9eebbaec0aa676aa16e2eb9e5a47f794112c7f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573832.23/warc/9eefc879ede368cc9dd2dd945936b1d2ed7e9d27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573832.23/warc/acf7b081ab2fcdd0e88794c68128e605511c80d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573908.70/warc/56f2d495b946dbafe6e41a10beb439af67ca7f2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573908.70/warc/8d72af45bfd688154d124447ed538668a42dc137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573908.70/warc/b9f2ee0c3e3e26ce329bccf3c5d40dd28e254b05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573908.70/warc/c2132a04a52ec5d740a1b23f924f7a10eb92ce1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573908.70/warc/dd1da9266012864b9a29575804d3b854b5646b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573988.33/warc/268ead563d25eb1210ec2239371dd0b6c9f9bb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573988.33/warc/458808362bbbcdde28e9233dd9fe11f354979d27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573988.33/warc/687fda515d5704d7f61d4b719d0c55d50d58aeaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573988.33/warc/7e4b174e17758a99d2ca75a90072332159a68741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514573988.33/warc/b7addce28c41f28702542379ad827eea7408ffc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574018.53/warc/1fc5f64114f445e1f498a785b82ff2ef16dcbe0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574018.53/warc/273864d0c445846b8745a39c66316a60bb3a853e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574018.53/warc/37bd40356d435a4423f183cb57bb41ee11ecd5f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574018.53/warc/5977d42c971ec31c6956dc06d5554a88799e7cb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574018.53/warc/61852764f05439cfff707b7bd680b0fecc97afb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574039.24/warc/30b07364927e442c55d5cb6a94859cecc8ee5575.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574039.24/warc/79c9438d00fd026a89c2d4c7d55d43346db7eac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574039.24/warc/930778b887c650ee20503d26407f651fa64d27bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574039.24/warc/d621de902cf0ca0255ff9b1ac500f90f42a1b821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574050.69/warc/22347ec2b392401e72372c085ca7423b5d714656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574050.69/warc/69a39e311cfb441da3c0314c695ec6567088051c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574050.69/warc/786e79c4e6d8b5f7d25f89d8f1a15ba777655ce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574050.69/warc/9cd7be1428d58d908e47fa0d17f0b696c6cdd584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574050.69/warc/e99a9d6aca1e7dc4fa04b4e425ac6104df46f100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574058.75/warc/268533a5c3da17051fb74dec87f9fd6fffa9f42a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574058.75/warc/6e8d6ca5a7807db62db456738c188399aaf89b71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574058.75/warc/7e7352fdd6d801b7637c2d4941bf4efc35614478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574058.75/warc/824d784e4b9b3bcd294505be298a535389e8a97f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574058.75/warc/e3fd4cd195ccb3a9866b2227214b360947865d3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574077.39/warc/567f0ddbec7a0cf8b04a6b2232e88cea5f079aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574077.39/warc/63b7c7e2ff7230fcc6d63a202d5877647b7f7d53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574077.39/warc/c0a7d58fabb2c6e6c6b8cd733f1780a4853671c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574077.39/warc/c59dde39d59bb4429661c6943e8ac62163b8f7f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574077.39/warc/dcf491e43b47be5fcf1a2d4cc9b5f452b4f78120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574084.88/warc/35030bf64d40e325210b7d91b3ec8eae90d8e383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574084.88/warc/58f0464e99f2ead2a36c1e9e5713e68c931d2984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574084.88/warc/5b0d9a4f479a184290ea6524873a29330bc3dfec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574084.88/warc/65dc61a540c12bb765bd46c3da64950837f16625.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574084.88/warc/826bb097c74e222e6077c7bfe0d971faf516bc6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574159.19/warc/1674213747124e4a11b5a2271685431245a7b73b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574159.19/warc/8efcae9a8ac30d2c6140bfa424383276f355a129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574159.19/warc/9fc1070144e432071507f7ba950375ab534dba43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574159.19/warc/faa769498efd12d03590e0550bc935473a8df0a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574182.31/warc/4e9404afc871b74fd610fb6dc25c40aff510fa54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574182.31/warc/4f1b1ef706b4d0a160e502064c1d729bc1c91616.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574182.31/warc/57733ab72bc7ea5787e1e809d58c319437732100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574182.31/warc/60c966ff2c00eb53e5c6336a9cc07611a673472b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574182.31/warc/df420cc31957b1fb1e7e417ef4dc4b78227c0771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574265.76/warc/712b9a140912358b58ae7c6da47a2d5a818ec53d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574265.76/warc/731bd4b088166ea2fafc7636f33929a350411526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574265.76/warc/a6bd59b82013bd511cd096eb882a2c77c75bfc16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574265.76/warc/a990c1258782054b0ee741e7d2447dca64911766.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574265.76/warc/ac2e3178fc67705dd0e933e2f5cae5ef14542fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574286.12/warc/842d45666cb4b7a6fcc608f43cdad9b3970f1d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574286.12/warc/8d8db538f1821b08427b58d9eb42a51ef8af1e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574286.12/warc/b9666aff31d0df870661dfa1337ef30a5376e401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574286.12/warc/c055f2a17de24dcdf2530abd3259e89c44938f94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574286.12/warc/f2d31c5d1620d1f6a57eccad4b108aa1319fffff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574377.11/warc/8295b77e52b071a1db8621d13014ec90c695dcbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574377.11/warc/9190709ca11f71476dd1b1890a0adf160849f50a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574377.11/warc/bc7c6b18cc62a496a148c42761f150a793c8df88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574377.11/warc/e38b77f23401b1d78d8133e2019381bd1c0485ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574377.11/warc/f1ca2eab9218141df55896782bf7deada69b75b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574409.16/warc/20340f54d2ea76c100e423943a6ecbcad5c3501a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574409.16/warc/37badf6a35640c46e4cfb6349b553e7e0483fa52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574409.16/warc/a3aecb7d014526060c5508f2527c026350603411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574409.16/warc/ccb0d6f5a3206572263e0272d2ecd1a38167feb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574501.78/warc/37a7b92dcd63f8ce7f6b6587ee86b47931e08555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574501.78/warc/585b385c227e88ce81e97648f0e43bc09cd38136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574501.78/warc/5af7920053ca89ff885cb8d92391eb6ade5107fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574501.78/warc/be9ec9aae41d70e29b67cf80d7a73e79cab97bbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574501.78/warc/ea82577a23c428639e75060fb047bd0aa539c8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574532.44/warc/0fbf40e7a3603f7e7a571f1f2f10174507a9560c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574532.44/warc/4f8067416644d9b8ba3757af1bae66f9f1d668ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574532.44/warc/583f1c1d7842347080d6ab85ab5a0cf709e1cc53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574532.44/warc/94e3ea438a9cab96a8c70df70541d64176637b7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574532.44/warc/d14f066b085fb447fd4f7ff6886b47153bd9e79b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574588.96/warc/8f1ff1c83728768289b5c2304eb2c3c2de1e0172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574588.96/warc/b71638d2977acd2de562c27a04f3747835764ca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574588.96/warc/ba17ae432ee14aedbab24fe81b9086dc8e6c8588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574588.96/warc/cdd342c5890d0962a723139c4d99e1c8cd9c8e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574588.96/warc/f9439beebba668935dba4477be97266b846ec757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574662.80/warc/35a13eb31752060bea4e7f65e84936be89b0d1fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574662.80/warc/39ca68f7967ae543f82acd1127df73ded6267ced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574662.80/warc/4b6d717e482c055138beb1bd2710da8c545a14cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574662.80/warc/85110e20e17a3e4ccc6a7d54e30dcf20bd88b134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574662.80/warc/c942c186d4f3b8d6860ae62dc974eaf3e90a2754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574665.79/warc/441d679a9abab45171a863e9bb4facda2246ecb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574665.79/warc/493ca19a6efcadaed48b7a7749a43dd0f4685745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574665.79/warc/84abc573210e127f05e3a280be2448d75d343899.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574665.79/warc/9a4f1dbd77fc2a05052366c5773b416d9dd59e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574710.66/warc/05b4f8991a227ab52fbca1d851d855dd4b01ec4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574710.66/warc/541c8155c65d75e00d9f4fe67d1aa22e54ba7140.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574710.66/warc/7d6b3dd3708f581c8652227bf75f7e96532b4385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574710.66/warc/8ffeaa51e8fbc9a33d74e2486bc2a61dd7a6964d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574710.66/warc/f2cc380cfd7dc1e2d0a7fc402b0bbf69a3b70c8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574765.55/warc/34dfe5375757a5d631e60f38db96ab485e8f4f2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574765.55/warc/89c90c3a2ee41c5eed6026f52cad42af2fafd7fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574765.55/warc/cdeac2d9da335fd83f6e281b14219bcbdcaebce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574765.55/warc/e23f79b48e2bba6106b8fc96c988f44930b9d8bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514574765.55/warc/e4e2258d05d4439844971a46cbc92417704e5c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575076.30/warc/2a7e4ef7ca036952b8c69f9976cda57b14d6d4ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575076.30/warc/a704ef077738d721f9eba6e96fe857d0257b6507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575076.30/warc/aafb658d283a7af3f734387fdc1575ff75446767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575076.30/warc/adb63a235895c3013ae3adb638eb1ceb39fd194a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575076.30/warc/babf35e7dcd11c386856924ac3096c6fa148f09e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575168.82/warc/03f957956c57f55ca78ded3a2c3cadfdde081c5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575168.82/warc/0bd779d44af254d119165920b2b98f8afd3a0fff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575168.82/warc/1859248513b29447b9c99152f7d2af2a3161deec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575168.82/warc/ab7390c542eb7c04aa38ba7e832dc3519cfe1b0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575168.82/warc/d04d4cf968b5156e91409fe8663e241f12639ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575402.81/warc/98cc9460a20f15449ac55b40c96e7a2130685da9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575402.81/warc/b36ed71544e0e7c2a48ae4027cc6af18a0e8d56b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575402.81/warc/bbf42256d43e3d1884a14a82772f0b175757edc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575402.81/warc/e176955a42bf556239fca29e52d9632b94a9f611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575484.57/warc/4aa752f9a877cf740b7387ae5cfc71f5ad6556d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575484.57/warc/4f1780e4eb5a0251b6ef752d30beabd2e2e195cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575484.57/warc/acc7c571d2adb27b4f8a311a39c4228e36807f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575484.57/warc/cd00bdc27ed8e866a5e44395cf50626ef498cf9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575484.57/warc/e957331a5efbb7373def46d6d8af55bf3c0733a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575513.97/warc/10be8691c7dad64505b8406d6972443acd34189a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575513.97/warc/245d5d9b4537a261b1fb694ebe57c5e7cefbfe84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575513.97/warc/41eabc21123f572dca7d370463e50db97bac0a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575513.97/warc/6d77d2589dccb01c4eba8e7159b8f921e083d3af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575513.97/warc/9a14ee0e62ed0f1d08f1d62d8f5dbe4b3e1ac14c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575515.93/warc/1f032a621bf6a520e3f53fc47ff2711831d67241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575515.93/warc/c37aee78736b86cec8738acae32318053a189bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575515.93/warc/c89df2fc8a05b8bc09778608da60964573df229b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575515.93/warc/f6cdacdd70a688fd4c495daf2ad7444407788690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575515.93/warc/fcf85ae657a3ad2166a095a24d3082188d81ac29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575596.77/warc/07df79527af78ac8063694ce00674fc25061312c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575596.77/warc/3c78c97071a1aa6b9c9256ffd57b469bbec79fda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575596.77/warc/849759e44bb0711b98f610a93f6467cd494f5fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575596.77/warc/c4e4c64f8b428c006660b89eb3b1fef8ae817b40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575596.77/warc/d5e2fd73d3f0212087020ff23601e9f472aff1fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575627.91/warc/243fd0df99137c5930383d477ae355744963cc59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575627.91/warc/6c0f3728cc1e00b7cc6fbc7dba17ac0cb51b1a42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575627.91/warc/816dafb6bf9ddb976834a5fbe7810017aae5f058.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575627.91/warc/fd18411e1d43f376d27ee4ad3e0f767457a08a8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575674.3/warc/0348f5d784da7ea830b5316ee508566fc3af52aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575674.3/warc/06b710e26e7323dffdc1273331f24c54f6c53f1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575674.3/warc/2588d101e56d66249ced7a0ac7b33b6320c3e954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575674.3/warc/c28a960a085bf0dafa683dfb79715a1e95d0ab26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575674.3/warc/eb5c83613ad6fdfcbf3f5cd6a2f1fcc7b00b17a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575751.84/warc/787a57b5aaa5f58bfe18e368c7474ba7e75cb468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575751.84/warc/c374970c7cb29822e950fd9d0ba7c455e00e4f1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575751.84/warc/e27214a1f20b9b6647a92c8073ddfc38b081741a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575751.84/warc/ebe841c8c7627d8cd55f3c423b51b820ae3ed827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575751.84/warc/f2b280049e0ef60a577152eda31a980d2bc27a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575844.94/warc/052e57e6617c0ae1905a2d1c87a8fe042935e25e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575844.94/warc/2c27cce03876b14b636df598d688ab6ed7b5eee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575844.94/warc/34951997dd9342e5d217b0940dd0502952425e79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575844.94/warc/6d73cd1c8f3c724b3101d1d55994c10cfbd86ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575844.94/warc/fdafb32385c4a0b8808775eef66b2b267f3ab3f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575860.37/warc/374991dcc799acd38d3815f4b9fbb2d7651de951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575860.37/warc/5f27188d184880430dc7d8ef3376289d5a211db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575860.37/warc/a6777144c51d1a5f87ba3a050faadd060e0555ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575860.37/warc/c51fee24fe4992fdd7dfaefc4f61008a33cf6da5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514575860.37/warc/fe056600800e2e5e7e53c550d5c955df4dd0ec3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576047.85/warc/34055d9cac50b1c2fc5dfd13a1b95d4f0bb4e355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576047.85/warc/54492213468a1ebf0cb2b97608f2d161756287b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576047.85/warc/d5956fc55b62ecfcb951ebc9554bea1cffc459c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576047.85/warc/dc38a3ec87dfafa469ad50725f3237d7f5a35183.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576122.89/warc/20c23a735b1108072f521a28a5074c110fcc1e25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576122.89/warc/6e5d66b0364c398b87591c6e022301d2d820e5a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576122.89/warc/7c0dfa4a8efde6dc264ea23f38bfa4ff439014ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576122.89/warc/c404318a7c25eccad683126a53f619c77a758074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576122.89/warc/e4ea55e61dc659f552093475dfda700cc61af699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576345.90/warc/591da7e1045d95889582f034c530fbcf3e2dde64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576345.90/warc/6376a20c5c139488742f0299a616d3487eef51bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576345.90/warc/99f46529d4af821d9d3b88750fc6b23f0e98bbeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576345.90/warc/e22a32713814d26eaac82a172eb7065b5baaf2fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576345.90/warc/eb72127090f1e35c4d8df91313882b69dde57aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576355.92/warc/15635bbd9f4d93e8290dd6ab10300250a9fc7307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576355.92/warc/23a75b9284e2ca62df598b0b773e920cc4814b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576355.92/warc/3e889fd89448c04ce4538c8b506069b7c9afc9ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576355.92/warc/c01297f8a735fc5105bb320b3733c33dea90547e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576355.92/warc/eb87108c6863c813ee5e206da6ef51ceb92a1c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576965.71/warc/268a1c7bb7057675e147c0cb74cb6141f31ec0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576965.71/warc/bd5b61ad35e94c4b21f0b0cab195e9f559ed1241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576965.71/warc/c8cee0751f3634b83f5d4273cf7ba3f825de6a68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576965.71/warc/e55349ce49f60db40208107d8c4240eecb4e5c5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514576965.71/warc/e939cac9be3838e3dae2f2eea569dc298f683bce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577363.98/warc/2aa2e29dd8d787008e8a26b297adaad46ed1b67d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577363.98/warc/4a53ef635fb4e447b7a438058dfcf77dd52c5757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577363.98/warc/8fe1089608f1a521012591a87297261b97ba275f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577363.98/warc/e9b93479f83137cca61882f8a7ce3746db43d6ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577478.95/warc/0ce6f3c5b3ee02aa4eee62f096310056d91d85c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577478.95/warc/4571b7126b2f3121e1612e868639b3bb5b0747e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577478.95/warc/865ad8d0c08809efbb8e6389ad4899641ad34348.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577478.95/warc/86e25466c50c66823467af955d58020c634308db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514577478.95/warc/ec0c50dfb089738654c59414e89af8ae1b2d95ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514578201.99/warc/1ba80b57133cd7cd5da73e0c70de131af77e0373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514578201.99/warc/3195f8988f847fa5356249b07726589cc70d305c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514578201.99/warc/68391b33d2a996c7c72f8ad261be837ba9f15fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514578201.99/warc/6da48e0a0e36c386fa5186facc395b91e2a2fd26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1568514578201.99/warc/e3bf4ea721fb78ae81cfdab4640e209d87399c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1fa554a75fac1e825c9e78e0081aa741d841e630.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/1fe08670b4e7a6a80c3005cde7b2c835e316bfc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/27488a5d4a1d01b1f62ab4fdfc9b490eeb31cab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/2be5c2f7a61cd717ea6cea208043ef2c7ce69d96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/2fdb17278c28cf89c775d6bf809c2eeef4134b40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/34810a7971714c93e22a7904a9924337e2850bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/35ac5c85300658d2ef649f3a83eed765f9a0c28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/3860d7d6a987309a0fec7b3c7ce726a42b537fd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/39d963aef4b692f42c48b5ba3680e5ac87ace2fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/3c4ddb162f21fb7793ae6c01b3cd41a3a7b3f4fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/3d669767c7ac4be36a025ef9183be814b4c4eae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/42f9beb1ddbaa70229629b48434347f1413ff068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/4a39464ccee01af8751cc7395d1735509cfe7027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/4d541f4704becee16ba27a1fb3011b398bc9b597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/4d63c42a9c09aaced71aa5450b489ab5fb1ba047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/4d8d28f3c2e628cffa3a6d3d1a2980e89541dd27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/5795b6360fe2503ec730b518af28c54642b2248c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/58502b57ca43f74a5412e9eae23635ae067fd6cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/5aa023947dc899c6174c8567857c32c5a7778b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/5d9285185f60e7d1beb4eae78f0c1257ffd31f2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/64643f771737c639ac10e054d04fe4d9122db136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/65a678506c2c75523cebdc32095f23ec4e216653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/68db5be0a972007d939998a9b2a22ed48974f1b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/6e274cbfab08785bb54b4ad2f0e72f93c5ffb2aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/700df1173671613dbd4521e7e90df6ecbd678ce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/70c563c8c13b396ea9a1c77adc0e863ea6f7c465.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/7652eb523a2b0b8af8ebfd3b15af68a03ccffdae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/7737339d6c8ad4598e605264bd02c927efd8afee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/773ec478e40df1ad8cf2c6dea8dcd79af822295b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/79d6ba65fd95f4f24d5317198504bd861c74f14e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/7aa12d39f90cd03d4d1f51c988bceca74e3adfbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/7cea39d70949eaf712b86f1cc8aba7dc50920d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/7f2f180dc26a51fab1ae151d317d32d45ad7b2bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/89e98049cacff7d0489d86e6729f9f1981a6ea0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/8d449ff0c96c45f31f49a5e7776ac99da5026b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/9efe24cb56ae960011aac1ca595fb82286ae4b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a13f7d1b4c35cf10c5368059a1741a56b64d2dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a160ed3aca40c649e981f70b7b2c4efefbbd6dff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a17275ce3dbe01001802caf7bde07a1c19c9011b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a1d3396d1b561b089a4ac01e7d762a4e3e558f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a3888e79472b35542d969fc80845f8b6dee25178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a38d31c137d7312d249194a6fa442e2a1ae482eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a6dd1c468e5cea1d5c4946801405bafa7a2d6b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a82811bc2ee749318f0f30e3562969e502e35e20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a8da2cdfae4565d408f86d1b4a5f72547d98e640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/a90b316d7f8ac8dfedeea1d5933435a23c58e1ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/abaa8b31d1dc3b721830ec52bd61665c8120a5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/ac17de4886c21ef92420154828686f99168e4a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/addf880dafba7066056ae68412105e033d6227a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/ae39df68f27fb5938a955fabe4a5f8919e88c965.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/b14ea6e51e297f7ae1fc2df09925549679f52085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/b2465df6de29274003161b8cb58e814be1ea3a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/b6152622730ccf003d98e14e2fc59ea543ce36f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/b8886a2ca454d906af720b08e52fac576a2165de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/b9f3fd4ffde374316392d6eb9feb1f2d4795e70f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/babe73006a6b7b11afa45da5a1860e928ab7baa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/bfeaa2bf5c92e475f190e1c0286aefecb970dc85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/c230f40f6389402c86a5de61e2cd4828fb564acc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/c43a7a13637b765f1d9177c476458df0b39a2d87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/c660a8e3370c90a1e928aee3e4aab915da0a976c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/c7d0d52923ab3dab0d94ffc36eb07a7f88336859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/cb79278b7142b902340069fa81f102c10ccb3a11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/cbfd1365d24976e73dfd6ed2222bab50ab0bcf80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/d19868be06020cb1f9b55b15ea3596d17005f4e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/df5a1ba139edc74920e5a816f2a53c1ed6ed41bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/e50ed2491d906021b7700d1808ca5aed81456acc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/e51e03e56b743963c7add253fd45563ecb8230f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/e64ea7f978a68e8b83f832f4b9ba74cbd1cbd1da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/e7255c7c7ffca79dc7bceedb21ebdaa266bb3333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/e87b9faa8ed2cdfa432d3a65b04fcfb8974d4818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/efaf18228acbfcf0b81ce52f6d889659b233c41a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/f0a8411140f42d361332bbf66514fe7189bb9887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/f2684edafe0c7753de132b3713ebc484136039b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/f56d71d977fef81c4846cacfa8a4b4357f6ae1f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/f7d0ed880dda38b98ac32add88f716810a737436.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2019-39 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2019-39/mix/input" + output: "/tmp/cccc-CC-MAIN-2019-39/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2020-29.yaml b/configs/cccc/mixer/CC-MAIN-2020-29.yaml new file mode 100644 index 00000000..040cacbe --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2020-29.yaml @@ -0,0 +1,714 @@ + +streams: + - name: cccc-CC-MAIN-2020-29 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/23c7057a58b0ca6927492760e738c6ed07b2f4a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/63777e8b14ea278e58bc114dce82b2cdd6124f5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/7eaa61aebd20d0ea34d1f3e6c3922023e72471c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/96f09a3b58ca5cade977d0fe606593db3165ad19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/d93f451d5a3a93013e8d4c98eb3addde0b3e6e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878519.27/warc/eac1d252de63dc4b7b9bc4c116784a455373350b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/0cf2fbcbe4f0f8068f943c3918c802813518a5d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/11e3ca8e4265d2f2cf2ebd78ebb8484f26faa013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/3d42c01ef103bfd1512a840d060f840d143b9649.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/95e5a76081d66ac11b828d21961dbd88a8766f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/da5fb3758936e0f92cd17273e9b51f1d0cec25f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878639.9/warc/de286fe46abe46928154897d065f38635b35ef7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/0e60cb39049cb1909675d9ac79fa58de5affeeb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/246e2bffeb1b3fcaf64895a7549bc06c1888dc7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/355922df48b2ed799ebfdc7aae65e41066d20dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/416057d0281ec55cd7fec143671fb05aca795a81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/74cb5a8683540f7145a7a1c5c9e07aaa6626013c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655878753.12/warc/a70faae53c184008f772210b85359c74af48eee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/05ef949e367a6044b12337259e90b25e9032bfed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/3a301b6e2309c70f4c6f720041ef4cc1ec8004fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/b43292e0a9060832960a25e84833eb4f636854f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/b9e1221a55b2ffe2ef97ee7d4f1080966408099b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/c1512e5598ca1f00d5314a14c9cf35d9546cf7ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879532.0/warc/dc4be147d405258afcdce789a045ce03c3df4562.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/32623fcee1818d421ae2ada7da58401566ca00da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/58a63dd9a4a98d080d56eb5ac0d680ef1e95f64a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/898764d6c1e33b37299347ac1c5477d93189a262.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/b5467cae8112408e3ffe84c7df4f14e7555d2727.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/ca9461a2097c10ead0b7a730942f4196a33f02d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655879738.16/warc/f87ee890a15bb45af2d3bdc09fb16aa6f9afd247.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/09addfc0100e8de179fd464ff8d92dd14fe39f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/114abc9070384c73aa745283e09dd67e60729bcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/3b8fe21a5bc6a97d9d178f7c137f06090baf96f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/61ae775bee74c199582755a10045eacaea13584f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/87335305d2a6bf79c7885d3c571a8f8a75a25de3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880243.25/warc/e7fd32034e6abcc769c30f5f5ebba7cc42d91504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/280da6bceb9af8b0139cd094c3f84381bd7c1627.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/295eead51ec6cab0edfc93bb0ec2d12c22b1ff74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/3294b4bbe39c0a9a3a78fa518a1d8f0fec317b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/3b31d245fd556364803c83918223979d51dc44cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/901b70ae1cc3c95b5fb19bd23db57bc293bc29df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880616.1/warc/b9b183385e0a9c47d12190ac582e60b18b0e8532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/137c08391af877ee6fd5a98aea619854e18b27dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/43081661e222b09b31e8640eb34ffa1f724e675a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/4ad80e7598f08c9e58795796a33920ff5cad09a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/5d5753d6ed3555992e78a568af005096d55f1caa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/77cd3958255fe5b03224f2c5422b35e5c38e65b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655880665.3/warc/9396c08540a6911fd6c89206a4c68cd9a83575a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/15a7e710411aaca6f134143541cda9cdbbda7c2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/1d1f0801af1241057f95008a434982b00c3d55a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/4622bf1fe82aa422b5c86921431de3f3f68689ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/53806f0570659747ea7369327fcd5f99a40cc1f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/97c9137007e79b3d60bd65bd7d978e1ac6454409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881763.20/warc/f9674490290e5cf7c025f6293d5b6dacb59e3ae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/07e35609ec90ed4fc56d2e07f57682c457ab6b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/1dfdd1fc55e3ae2664530f40161ff35fedd5d404.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/3860d8ef34b71eb5eedd98c9ee6747faa5f708e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/56736dccf5482f656deed2bce9dec799536f2748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/99587bb82b8696861f6e30551cd3a6b9002a6101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655881984.34/warc/ff5e7d56fc2af4fe1ec00531c5877e5f6adf1170.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/044811eba5dbcbcf53ac23fbe77b40128dcdb0ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/06dfb1522ad0a97a38843c55062c798fd9d36ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/10db5b5b398a6475cb99d7060c9f83d74574c4be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/2c59aae43dbfe134d46de60bf455cdba8b0f1fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/3f21e3757ee879a2ce3f32a478efa5459d360565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882051.19/warc/5b0b2df48af3446c297a4ae0437669fbbfd6bc3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/030ee4a0f05ff1dbc798e31ab9efeafb03e97524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/102b2991198caa1081ce26464c46d3b7b043e325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/4dd796fb3296c4ec2028373bcc42be50f6e5a4b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/af2bbe1d78df6013ce75812a040e0dbf7134e242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/b88c5456cce1438c16cc90a7658cfc1d210155fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882634.5/warc/e0c2e6fba422621a612f9a10e7770a4fe5d336e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/033581c9209383643b5d911c18b4987740c9d8cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/59b1104929b8d9a2f6f1c6631a81371749ae5ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/6ea41ed74e57ba5c58491052df2b7e11edd280ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/838bfb5ff7513b8a714dabfc225fab24e62d2380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/9861fc1e82bc63bd955e6529599e0a292d1c4650.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655882934.6/warc/cf2540e97956254e432f2fc646ed805b042e4133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/0c4afcc6c3a0f5bd833a92a80ed7103d18fc07f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/131baedf3b7633fc49823ef43445b57ac6e527bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/74f2a9652019d65ceb221a849fa72e724c5986af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/9823eda2c71c06c2fa4650de036ead6516cb7232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/b8497dc3a33ef7bcebea8cdb92ab87deb48e8d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883439.15/warc/f6697094e35a53aa729d7f405bf68216865bd2ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/1265b6a8864ba8c9e98d99c1afbad6ec326a1d7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/1510c33373c40c0c433327026f647a3450c1e70d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/c88b4e6d40285151e65e48e7479b570a62684bbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/d11c916fa89c6e340df393e9f92b799cc8780eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/dc40a3f95bf51c9c300c5929be6806ed662c70f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655883961.50/warc/f7d03c9468c8bc540411dc92198df40859d5be36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/14cf9946154805cee761af00c7f445f210b16099.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/8018386c4f066117fc017982477e63f3f73f6452.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/8b245150c92fd17d57d3a8a4bc6a9fb7b107c3ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/9bdefe52b9a2416724306794fb900249cef1d7f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/c65cbf3ead4c9282af8adfe3c91ba646130f3be9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655884012.26/warc/d8fff6d5936ee4f4c360bf169458306c341b760a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/0351277964bde7cf22311f267d40899d8d80b29c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/0bb49f1d1a89733e8cccf2474eef51a3b62da754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/362ff2c970d7183356a84ef78cceb611b983d3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/400f556b75be4dcd715cdff8e1b09937b0fcd0b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/60c893b642fe41255578b5292abc983fb43d248a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886095.7/warc/9d9f2fec0ffb08a2e228d57d71fc508184d1dca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/01f9abf30eac615274382f90f5a7a98fc55440f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/9526f5679b09ac0ec7aa7f145d35002ae4032420.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/9c94d099e01402bec8a0d5837ffc1960732e352a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/9f5ad613fe4efb9aa711084ff22963176f35269e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/e6df4770d78853ac913ecfed10f4529bcb4d359d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886121.45/warc/fd6c35a0760b56afa623992459b50bbc10dd8e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/1957b15ceddbb793ce6dff32e4a88ed751cea417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/97fd44d14e5c43bc78567166bb1bd99f5c92ae86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/b36af9774b82d2d3e58c92bb504b2bd8157f7b6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/b900afe8e1dce20b180b7a2bd9b7a52b778843e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/ba801d070646a61e438a9631ae7c15fba8caab41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886178.40/warc/eb7305681b7f74522022fa73ac5c0ebe37901393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/07d4dba685b2558e4d40730d05e4b6f912961c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/1a4cc5a6ac6e10d9962a8dd0794da91123e3acea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/8292f9f248825bab1e496835a81c23fb6427000b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/973570265df0d1dc7ca7ea4417abbc02adb012bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/aa7fe0a1f49d21f11515e06c29b0aba5e1819eaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886516.43/warc/e559eb01eb07c49aa35df46e7927622aa5778381.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/0a0c420b61b3def1bd9ec6c42fefc65a96db4807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/167b2910ffab03de413b8d8a8f50102c7745593c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/5a8e375a6c798a5243a2a5bdec5307df0cc10c5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/6a5cc8f41e0eb3001fffc64fa0eb2826ed3e435e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/8f67f3a273b8a869cdf146fefe0c6874b5d17809.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886706.29/warc/d3350cac2d34297a86d4152bc26131fb8038a5ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/0217d2c58243f4c589bc2afcc9f5708504dbdd2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/073a0c1a8ce655658a113a08e6396455ac44a11d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/77e1d81e3c7f098871422b4ae6abff36aa6e0d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/9e41668644d2b1dce55ce53cc4bcd5112cc7983f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/a29723ff4d4e0facfa431bd3f57c251cc3344e19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886802.13/warc/bd5484a2f7af690680a76e363fd68b61fec561d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/0dfedc9e074073f8a9b209e70cda10397245ad31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/1de8272ebea9ecbc325b4a0b25811504d1b2c992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/618ff227e9af3592f0b1399224ad55eaf75edf67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/63c0eb94a8b52a25ce26225eac4bd616ca412805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/ac1e4abd75d5bbef224e5f3ecbbcd62c407ce429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655886865.30/warc/fd9254868c5b6e7e8a3cd45f268826d97d8feee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/0722356a9dbf6ec16a14e3fc09c2fc591a6cffaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/1a0b68b248c34f6efc5b5da4c3e7b852bb04d8af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/343c7aef7e721b3bbb9de837c8253d3ffcf46087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/5ac73c7aab5bcf56f5d0d4aff403e4bf7ecfd90e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/cee06e7762d630e894e13e15309cae7152ef261b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887046.62/warc/dce33b03dc20e8df9ab23adcd842838d2f19c5d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/0c9ef972a76b9976f7e9b92f95ab39e0913e22a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/2128c1a4648341c739751b2219819b149477f3c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/7f891e8c9551eb5cd0ecc6f2b3bf5d4bd97dfb18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/7fe171cd3291c2602064e675d2c46eefe0006c0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/d29a15b3b89defeede2d137d8a939105f4e3f15e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887319.41/warc/dfdb466b3c0af0c4c90b53c09faf924f6e254d4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/3005ee5a07c1ea8a599bf0ccf8e0f886d6e87f3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/5785d61a0d927960aab8caf7f3e619670cfad2fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/7ef1f88688edd52a4b2a5e2b45883a7dac2f9820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/ad571186892a32011c6a6669bcc8db56ec5c8264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/c7569ef934bd4bca24d6e4776faa80edd0f5dc28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887360.60/warc/fbf78a42753169a0d5bb1e5b94e3a3245dc9825d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/0bb09a02387d314257be0690ff6452b4bdcfb275.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/5d5ccb00baa95f65ab3d84f6dceecd49bb87628c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/6e1745e12d38e2845594a715902ebce6b3529b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/992aaa4aed9190ca712fafa76183b676cfbee911.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/a9c6aa46e0d23f178822a1994a771cbe66d06629.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655887377.70/warc/ce57cdb7bb2116bb9e5e2312e99bfaf5145ff6dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/1169f61b2380319e95cf6ae92949d25ca5e24863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/52e43b34fdaa3bc9c1a5c8f6120342cecaadb5dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/9af9af5bdc6946f800e03506f4fc03b6554b3d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/dc1ac79466b3113f10e66b735c60088ef9bca067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/e0775e9fecddef2bd91a5f34cfdfa05a4098357d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655888561.21/warc/fa00ed37a992f5d4b8a8b41f67167c7e785094f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/2aac187885af95b5a4436a6d945f73ea1d45f4f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/32e2755b73e9e078d3745cdbd1bdf232f90ca408.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/5855965aa148b80ec5b07465416b32426c3d71ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/8b92e882f58b59c3727942d7f60f5f666b98b042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/bd6945144de1ac39a33649a92ae4591c74b055de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655889877.72/warc/e124affbcb4362133dcc560bc390e15426240b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/45b2dbbe471dd6d49364c93d29c161b664ba78d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/a194c1abba2e97208c1152598a4e2725efc29e99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/a712256d71e2609278e1dceeb6ef27437ad3b482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/b2c71c4480daa14440fa4ef0886ec9a5b88ae00a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/bcdaae062dbf873b453d887d54412cd45ba52243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890092.28/warc/da903d463e687373a703cc631fd6209fa3d18b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/38e2284df4dfec7edec77b1822293ed8ec8b41b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/9692a1dad95b4c3cf3d9a982a3324a3db8df6dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/c11c5fa6958cf10a8a1d6f65c704d627298e504f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/c2931a8944cb41b8745c62f1caa194f1902962c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/f3e7abf1cb1ed2dfe85bff337bb6e0da02ddbd00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890105.39/warc/fb685eceda2e8308824b981abd0c585e30454ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/2598e11c423854dacff94b88919cb7bf7e5efc25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/720ce904b772160b92f777629646807a0e945a39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/796632b05383a6e9ec79a4e48d05a03fdd13c2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/9b4d7bbe49edf47f0509458d6f613b1771adbcf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/d1c3febdc62652a6c1d5e4cb051a9ed6ac38c41b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890157.10/warc/df0d0b95ae9fd9ed5d732d9aff1f3752336abdf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/1020530a0556d9be3659e253479004b5d61fc180.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/24635db4292749606844eeb71760b53b5572f4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/64d4324530684196615741c64db2ef90e296b6b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/a30eed8076c1dc84de8f7e81d17c5e42cbec8363.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/b798a5dfcb95b4728385bf3d8cd2ec896bf1ad1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890181.37/warc/cb476a247a2feb236ee06e6e174442b61458a1e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/0e346cbca1add9a1b81b11ec4c38a1744fa9696f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/145f92b017164051512a92fdb7f32d05f9a29bbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/a5deb3236e41e37312faeb050f2bc7724c1a6089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/b4f9dc5bc1676356600b2a520088b388f7fc3008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/be3758025a51e3f23e670c2771cc7f0e5fdcd843.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655890566.2/warc/da4e9b4b4f83b45d851f0208711c457a952c195d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/5148831c427814cd933d32b4507b04c544e7272a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/950447c3ec75e9e6db64ec1a3dfa6acc209ddaca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/97d6c103e6ece871d256525e002c7e8a05e98ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/a4273294bce2e7fa23656f0f8434d3da723e8cfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/f8e52907a6d6554a9e86655e437d59bde6bc2b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891640.22/warc/fe88ed9adf796236e88c1931bc23a760ef6ee3c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/2e232bf09404e8f9c4b3574d681a9ef71cdc99e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/39afb30bd0f66d38bbc21c1b0f5e77ed8144b232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/3a04c374845461da5e0e5ebc04d202c9ebc9c3e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/b04cc656847149102c08d0f5602e4ad39d7a6932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/b95373887280dbb25c34f6598e3ec6c41857aa36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891654.18/warc/d68325bce9dbfb40411aeed3e8fc796968dbd656.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/32105bc5905c520bfb5fa3a9dba857a7794e5476.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/56027f0616291a4ba470ba146d784e3920f532f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/6a5d7f1ac4f3eabfb070524a6d0d6fc4fa1a4241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/7a572b2512b4adf7e7f658f661eb85fc8fb52d02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/910b870e074bfaa650719b3f61d8acd5f8e189fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655891884.11/warc/a0086dd41914b0c8068c098a61db0c8caea0fd97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/06feaca411545b64ff52bc0bd9f94e80dcefc645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/76cb33cca046c06d963056027f02262ec800bc76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/8484aa4a225d741b82bfd9f1d796102098f4ed10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/86e4beedfd28e11d1584f799ea04d8f7c1025b9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/b9046973567d00b2dcc320c80f81dc2e499bbe3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655892516.24/warc/bf72d4deb696db747d027e3b277bbbd09a064439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/4dc366099d22f6499c8ea67ca9d28b299996bc38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/6d883890d92c27c9338541f2db1ed8282a8a3477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/7efeca2b5906da1b391ae27857b8331f6538600e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/931e71f8715498a266da3309a2d23768b1675748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/961b893d5f42e6ca1c15adea2e459677006461f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655893487.8/warc/9979013f919396b86ab8b8615df6215828ba1ccf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/02677b94b8c4452732d6bf86096d1942778bd088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/04e2b29829f299e50cff5b167b80855616ee1bff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/7ae729fc4148d73a012eebcc170b5246f35f9f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/9e4db355375a22c677d7ee929547c78d509e0130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/a227fb5f0778d72690ca397dc6327fb5ab08281c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655894904.17/warc/ebe7feab20a048db892c3486a9565c007ab2066f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/269936a865a88740b7c5c3ac8e09a2f4f3139ad3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/2f0f4e3b6181027c58b46de02ac0cca8c4a44a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/66aa4c958484ffa7b53b792aa8fcbc2abdecb0c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/6d216682aa703f9c37c5992e1fe0b6363ec5a98b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/90a9a35eb22954f3f86d6df5d4a4370b369c560c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655895944.36/warc/ad1eb86047b41d9c5511f0df85c579b06d890b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/2a9ab6f75330ddae4973d692b658cea235a93b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/3c1298bc1713b8b750cd4ff5b9e0b2284fd482d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/5d0f1c64371ab9a4c79fa4243adc61cddb669eae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/edf2aef761628c48e7dd823f2e8e9d08aa5840ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/eec779a2b2acd1f6ea6e94672ccfcddeb8100cf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896169.35/warc/f7a6aa1f90db4cc5092e915ce2b70c7b6ede1291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/531d9d7e9d2613a4943c08baa860a76108ce6c29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/59338d907acbe6687daf69bc380fb354ffb9209c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/5a5b6bd8226fc1d6629ffd289ff3c836948a07c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/b56232b82fb0accec1f988a366af4d0d18fecd26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/c0122c7e27dc6a1c249d63e0f0d243fe3118cce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896374.33/warc/dca18a1f92bfdbea9830dd17d3a5f0dabc5f7511.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/004c73d54cc058c87925bc8b48eb2b8967f9af51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/02f53923268448481711261574e9634b6e40cc6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/0d9b7624fb15e9bf8fa5bbece79bf52cae4e8054.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/28506a211f6b3c90a07aef569b60c10653eaf94e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/71d8b47ed310dcd4e3ae7552b3a3535a9aedaebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896905.46/warc/7f70d568a4bc3a03a8093ea62a07b8a8a1d78e9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/0cdb64d7547d3d594d58798b74f00ebade000534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/44276dfe4404c57efdee38c999c7e9f2bc0e298d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/532c0deaf6ee8f5953b6894c8220bca08c421944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/9892fd896f51f0b7f8407ce7e0ebfeea3002981d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/a9385083ffdb30ad69c635e6ecb7a315ade50910.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655896932.38/warc/ffd0f9acbd6327faff9492af3b94e69661513108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/210269ca04fcb771b728ae54fc2ffc42877902ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/2dd8aff9d9ad06d7fbefe8428a96e949d01e991f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/4fb42aa7cc9e9d584568f1270b2b17a52ade602b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/567ae1f55c7e47e15cd56605f469f24e59fb724b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/a3a47c330b0cb0b8e04391b7f868f21cf43286e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897027.14/warc/bbd675545ea3efbe2222c705e12f8a1ed4cd4db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/19fc6af9fa57804284efd26cd0f5f10c1f6c65e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/1a7c1160ed3f89a795e04af0e3bc71a1e0bfd7ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/22b86b861b9d2e77072b82124ae50586a026590a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/2d228d0fec573615cce7ad16701ea846f1911369.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/c2aec460d2e4be61d1cc53209f76576824bd547e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897168.4/warc/cdff2479cb733e2809ae28c668af0e1b76694a2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/39d39d3b4d8e40c54a01c2870760b3c8b3a0ae8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/6472134e0aa9f7baab3a20a296951704eaf0d330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/75bea127c4262e6e564a67082490672077b1fee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/e986f3a40b091eae487e5952f3816b4ae9fb9433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/f98ff05d4f023caa4a5cf2eeac1c1afca9f91be1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897707.23/warc/f9e8aaa7bf5d1c1059302af81389d163251d1b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/40e93f4d8f5aa8a34c36c71d919315c694911b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/4d06d51b6aabe5403770a049a4a0fb24d3d58ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/599e2a68094b5e5cfa446c655169246982529e4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/7198fc391463abde037e5b540005fea7417772fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/c1e151e0f48acac3efeafc348e0fc4fd28365f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655897844.44/warc/c63a0222c0982ababe478ef3d6870b9a00f38d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/15e8f846a4d07365d56760e882a74dd29516a4b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/6a4d0dcfa20fa9f1c15ef708cec4d1b2d889061a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/807b123bc2c75f2f78e60797493e686cddc5f7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/a0cba5e68165a0ae7bb617fcdca57405fb0c53af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/cbe6349dc2809d8c58f2d90cf9d86f3bc0334d7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655898347.42/warc/eb2abdd5f6e31b6a473286d9b4b5e3c0dbbb90f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/5a547a62abb51d471252830c56357cc78da88717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/799a5ac4f77b4798ae29cee994546584dbecc95e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/9c5bcc4f6805b496d1b22afdb82912ec975123a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/a113af7606b1e619b893f515c24c62a4f7b9b9bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/ba4d1ad366e7a5cf1264f8a9279979e9500676a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899209.48/warc/fe537f21794641f557b664a6e81f15c7fb774929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/125dd0be1c8184341c948cf8713a34cb70eb4f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/43b61a075425ea918ba4a6aadb136272a5f778a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/6d2bac322aa972b63ccb235987493999359db62d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/9537c851a40a02718a0dcbbfb52323a4387b7d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/ce5d1f8ea2d998a50492e78114f75640c79f9708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655899931.31/warc/e83f41083e9667210c462f3b3b1c360a460da8f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/111091025608cc6bea0eb56c4bc0806ca158d5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/a287585f4823899e7fd587c3596aceb38b6fc27a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/c24f334f6715615c2efebf0b2f246f177c4076b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/ce25d56351d095cd8da18ed96160aa97874bcc09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/e93a0a8ae7b8aa4e02bd7ef941b5059912744227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900335.76/warc/ea3b9cbd693b4101d9849e7bb6e857a0f9869fcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/0bc69fa3cd6692bbee1382d760f8ddbdf69ac289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/1511114762323fed8fe915175fb7ca2627f923c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/67da3a863c7962d8788843bbf2fedaa109612d86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/6b269f460625c9d3a88614f7618fba9d04a779b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/d12bb304840cac44c927d31a7b063c35b25ad364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655900614.47/warc/e781ccad37024417c3cf2a79854aae45c621cbdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/10df3c47d7914b7a2a0950d51e05458549779241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/120d6b91c372f73abc09416c5cee9c496d86f1e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/4398ef8cca6a5e9df6f0c0cd43327947621c569f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/ad7db038e7fbcd62125bb2093f0e327d450db726.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/d2220eb7caa4c4bcf74a50d50de8d2352cc215e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655901509.58/warc/ffa4d85766007d8d54d8cb5038150bbc90f7647e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/25143ea6599c17ff94d3dcc96d06493ef90b485c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/30d987c120cfd2170911e02c42e8297c647ed7f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/33a477c139a7d888426fca614ea822b0fda822a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/5175d2496290aedf3c4805b705217576d683179b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/6114ea1146c25f9c36f80ea91f5dedb822cf5de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902377.71/warc/c2276c405997a7a66b35d4e8ada1702fd5f39528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/081e95ec1f7b9f3c1df6fc45aec17d77d82f19ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/09c74ba1284a66e86c66a5553cb0b0924c0e9e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/2e1115609fa2468306b4868b40c49fd5a196b4b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/5c5cf4d0d07fddef82b21ea15814d562c502af9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/871177fe116f9fa6f6a2de20aa897692d2199477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655902496.52/warc/87e42a7a8d30b4d6c2680f2601dfbf775778e0e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/1e0f20984c8a7c938dbf2cdbab04dc8eba6113b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/2389e10506f23fa12814e7d1babef5a0508b211d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/2491120fcd610aca9e45c04a55530140dc070e31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/33e2685f9a711a945eed6d5522a8db9d2cb39504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/6574a7a3f767aa1f7a19cb02ad2cbd5a2991e579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906214.53/warc/b51f8122b5fd23f7e82f2e83268268b8a61baeae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/413221c4b0c81386fcc8e3f56e0aaae196a4e337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/7d0a212a4e8b43d5b618740cb2f647db6e5609a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/bc8bbcbc26a0429cfcf83bed1dd819c6deb3db57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/c074486ffca1b2aaab05a296ac838c0148426012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/d1cc682352d5a6d9363d4b9c6cf56e2683a4e733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655906934.51/warc/faee040b57232a9dc2cfde568a8a80b087eea7d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/1f5f6e8e0b6a3b0fcd6ce9980138ee6807f1f665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/289e75ac9594ec3d20d49d98d61a384d5d061f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/758ec6efbef7c3141b4881427003365e30e5d4fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/7bf06fcfeecfafaec14cae18a8b4c16c97bc295f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/96440de4ee9e167f27ca14f40c351473c67f2646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655908294.32/warc/baee03fbb8a7646945913bcf8be3f6b781a34b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/6c19615f8dbc0b334c709f1e310b00f4cabc9fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/6c57580ec0585256153f98133a53290607aeae9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/8dba864f881c59d628e9edabd0e6837f2c68faae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/b730b442e031504aff91fc3cfa4872b3c56675c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/b92ecf389c0eed301d5c68725a10972b19d58bf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911092.63/warc/d33e60df8ce00d40cefcbd559f08cc9775e9e96f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/3b214da96af19ba8120b6c2f9fcd23d90e79591b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/7a3d8f0dfe2d5bdd37c48c8406d76ec57c6dcd80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/a17209a0f2d36d98f76213a00d74f2f0b734e164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/aecf2d1adf8e7ee9769e74bf7a15479bef34e309.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/afdc4f8536d0d9b336a1a71c17c2480bae90c015.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655911896.73/warc/d5fac455b25d1cf39ec9d20d5c5c9473d8900ffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/0b99441514b22b1005b63ee04c4d269792e6e5ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/1e2da7d0adef038fef5bd9727fc9b27b0da8c973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/3ea84adc29f35d769583a188ef0e3c27b3bb654d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/5037fc6dea1dbd5670dc7a8c1062796bb167dbca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/8e3ce9cb0dc262e98c818972a5c388148cbad9f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655912255.54/warc/c1d387c41467399645657287f974ab3a7f3862ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/0548f79359f52c7f00001d04447be48540c98159.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/11d35cb0bfb657b43439b038a22370eedac166f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/2efd35842469b0695239e1828a79eee89e90020c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/84cd4d4766ce253db8ee4410725c456658934895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/96251c56f4988d72598a770410f4008f0c48d29f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655919952.68/warc/a6aa49542c6ee5296c3d95baed399671ae5f5346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/0674964aae45cbb99064abf478375d2f30051ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/384d8deec0fb68fbdec6a138c04ca9dfd29e8776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/a1d7d557cf49a2b47f347aaba6ab5c97f0b6b697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/a2ac24985286031b688983c4e3d9d79553a1e928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/b9385726f24de872826dc65fd4652056911a3961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655921988.66/warc/c70d1ed44c6a76d8c47b9160a5fedd82185ebfb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/30beda96af3e7a44794cd875aeb3f0b32e068d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/4940710627852c1fb74e788a60aed54e80afd11f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/5bcac1ec2d2961f4b6ced78476c970d4da3b6474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/6e1a9b8355b1db611c641b4a1063983c71abfc3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/7f64db9abfe2464eb0f5568104736d04449393f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655924908.55/warc/c4f0408fa036f519facdcd168976237d3fa5dbc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/4e51d1c78efa833a703f9845b0f4385a4dc1389a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/65bc1594c2c8a7013faf1580638c5990fb087601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/707309470b9a7cb6cd19c755e9a81151bc36ea6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/8285309523ce8761bac995eb7f3e66dbb1b9ffd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/9939062e93156cc75def068e6e0211fedc2427fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655929376.49/warc/c70069939d48d1c10fd10ee8ae62d016bcf996d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/0be96e94104ebc814aa6e8c81fb780715acda0c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/567cb2e3f2d0a06b559fc607e99ec6681dfd0936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/5e6cda1d79329441c35c989c328eefbbf16a379b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/5f90fcee877b926270b472260670955c86a940d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/751fde6400ffcde1e046185db0f5a37763401f38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655933254.67/warc/919d90ceb2c4f6da6f03ddd26cc1c455ae548f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/16a2a7b036722503c41d1db960f13bde24aacea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/787e81661f2f0a699a1709ab0fa3d35e3dc54466.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/7ff4c32fc41da3a44d0b37136003a3904e90055a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/a024bab4ccb77dac3d4f9e3425104435fc861e40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/c720487da1d7c8a51180c2ae2a363cac196aa764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655934052.75/warc/f2437965a1fe388450640842117798425189fec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/0d3b051dd9847c3964fb38f38ec2b645e4192e35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/61a9f4f5525ee04e43bfac0d52021d7b698c6f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/a7f2c0895abb7282f56f0d9880b57b3315c86bc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/c37fe2a9893b5b2cd76810d7ae62ba47ed5c6fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/c60a16fe79767420fad42e4213a2c69de83a7090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593655937797.57/warc/f9cbd9fefc01d223197da225623faa74588ed4cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/164244abac0970f1c45054921596ec61a8f82fac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/2906fb097d1795693b9c1e2f4e8fab2352370b0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/3287517f998829f83e1c664d106234d73c9d538f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/33c2ef3b382590efc881f696237cd7644661ae89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/468223988ad6acbc2154e9d6e344b661fcb3119f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129257.81/warc/ae443df097114b757a908d18b359942b94a15d30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/1c255e9b48919deb5d1d2ebd91465519e146a66e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/4a722a58fea62ec1cce41c385366bf957f0ce463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/609c86443d4bc972a25bf183fe18e3fd431e251a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/78d2b80e2f31a4f18497bb39261556c1057e4e4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/96eb5a619cfd767796666a3b5d01a3c8a9817d04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657129517.82/warc/c3a795b4f811b09c1b6c919163ff11d02a08b68f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/61de7d3d4d61486741d7473795a93a61d9b2b99b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/6d7d41bf761efc0177b1c248c1a9e6e966ff4b48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/a654a856ef7db030df80ec7582d8cafe03b85fa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/edf4fa58ea1a6b56257d4a7f1221f8a48dd59e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/f5f45336bb7efb78d34a1adcace7ff0b8443ea8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657131734.89/warc/fca578f04dd1d71ec4f1a768897f465b73776055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/140a3cbe0641b561626af0d9a0d6da6d2c45ef1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/19a58326158f52eb7fe117249a89d9b53630727b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/2a507fa52d7622345b560d900f1b1141b481c896.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/5ebc0d0123d0de76e84e6e3df143e0c55d53c596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/a3d43dba6515b02ed1fc84ff8a38ce6f1a569b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657134758.80/warc/f998e3196b4c7db0335f2df48713cbf2a1e9654f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/01354edbcfd42124e7ae01ecb07550f989096299.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/7d6f32597a215074d51e6fd50eccaad464906f11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/8fc1a083f7d3c8bbf382a4b1220b0b7b44c250e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/9d897898becc1c7994ce103b5b8ae94fb9631199.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/adba12d919ecdc8fe4a70de7bb8a4c61027b2748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138718.61/warc/ee07fb66a5783786b273eec498d5200fae5f4ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/54f6c934bc6ef05703603412b7f021b2f94b6bfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/69207cad08eb0e57f7187ed4429e8507719e55ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/db7a6df2c8a2f08385f48f8f8af321ba1ee27934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/e636b1c83ec22a38f3cb0e99122f292aa914415a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/e95a6c88a981cc4f8a69fe33d90a10c412c6234d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657138752.92/warc/ed7e2a4782ab0ae3e4d05233cd52895f2559a49c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/3765706c9c4d49799b98c9b0cb9e7e3dfd79856b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/4c2586ad9dbb690ff7488d1b50709f6defec0408.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/506a50d46c1c5004d84347067ec5270814a8c581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/5d52433c6e8bfda933d3a8dab3b350aecbbe5a2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/d171055a08810dfaab1756223bfe41b871d477f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657139167.74/warc/d731ba397b7f9ff1f3d2f521b90f439537935db0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/01170d17bc6b1eaea1c4bc5a3c452dabbe5366b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/74a402237a7b65a97e9957476284aece83f6aa0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/abff5ec8620e8b62b0f057cd5a6e1d42f7d647a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/cd9d82e8a33ab713ab4187d1b8eab78227c8e762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/e8b83cc2a9072be6a9e6eabf47ad3ddc7acfcf01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140337.79/warc/f1fa7bbab63c92815c1377cbe942022e83b4688b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/4f93523c5358165b69b21fd3362666bba6b1ff18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/5ef14439fce2ec87b345a0aaaff300933f675687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/65c801e97db741c47e1f217b6b73d36d9545d2dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/83eddd83a235996ce7b1c6686ee058eacd24a531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/a0812a01467c1d694cd01f3f70f0b848ae852cbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657140746.69/warc/ec6e250ad290d853a8262e31af79cda814e2bc47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/9da1a02d7784d63ccd1d7e0e981d7aa22e7f710a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/a30b2c0c61edfd49a28f38ee0458838ffcc181dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/ae1cc401f869c2bc2c5afceefabc46abd9f5513e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/d6779b1eec8206daa7bf6f7d45647a578e49b21b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/d8f7fdd9258a5e6e72a32edadf05871a2506f806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657142589.93/warc/ff7ef48d19f96a9f1d61b04485bd33e11e05c70f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/3b8ccc0aeaa7a503e3512163a14bf466be0bc623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/3fcc8667f320f1ec999344eb8e144700594a7d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/5573fb1971eb66264139b625f6ba45c1f74c0330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/5df64cb024626e0b4ce51b069842417af44fcee0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/83033f47465bae721ad55637acdee7338f0bad18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143354.77/warc/dfa093050de5383d174715032c051b8c31ca720b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/1015ff4cbfdbdcb1c2ed65ddd060e61606e6d2ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/2c269160b22d35746bf98bdc3a171289c49509a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/324348db1fc5d2781829340600868383eeb07d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/6402f01f691720e24cb15bf3b5691d5c3ffa9b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/cb799c9db8b849346efa1f172351efa04e08bc17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657143365.88/warc/d7e6686861cbc14b40705a8746f5b12e50d6c8dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/16c5be3544bfd32dc40182d86f30cccc897a598c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/22a371023e061cddb85846f16807dbd1c66076b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/8b3903b4f8a2bd3c7f4e3697445120eb40ef5269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/a80ecc02a691dd8f5fca0e0f80503590098c144e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/f247b48b53de7f46187e7b2fff903329b399cb06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657145436.64/warc/f34f786a82d393b64aabc73c3547ea23611171b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/08e259087d0ee34426bddfbe611b35b9a423bdae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/095beef5c3c193b2cd64ce87a50a990bcd437db3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/0e06322dc1f1059efeb6fc275b0e6bae412122e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/54393dabc142dec3ee607d39fd833ef2cd681f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/579ab7e65fbdbe303af2cb789f9c427fa5794fb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146247.90/warc/b76c734fa415687936b57826e804a484c71074fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/29f7877f1d4d9ce90d36acc6395baa35d7d7fb1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/379f9886471ebaa50271511bf0a5520db1ad850b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/63252740f193b65b90f3b274beb3b71f71e0d3da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/6a23dea5f0b899060c002f3ea57bdee8cf6e2865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/c1a567f65dfae106c0e71784a3d479d820cad6cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657146845.98/warc/f36b8e70ce954afc38476b25571cb6fb9681a200.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/019276f10839aaf558503c068a7fb510355a4101.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/16f3723b04d2ab425eb8c2fd2153e2cf5a3f149e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/1db13daec4846da972ee3fe3ee6f8100f6dd25eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/565f53e9f93b97f95c5a888a505d22e85100de03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/d99ffd8aa3f82367fa75fd1ed03e4f42a6abf762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147031.78/warc/ef49e9184f0290b68bfba73d8667179e66a43646.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/4ce16dff37e280a7aec0e8f005efc89aa36356c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/a43b45132681b478d0cf5c536cb7f1701f49753a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/ad0ae798ca1adf31e9a706fe4b6b79224f2c41c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/c6a699488be47a1844fe37f550657cb1b680cb8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/ca8b2b8f163275a41276bc97681e04ab4351c438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657147917.99/warc/feb32d81dafec2c9a969a9cee2408a76aeb5916e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/0f2d1d6d3497958a99b6a4b313b170a2ed8dd624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/1222c2d91b85e29f3624aa224659bcddccda9aff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/1e31aeae00967dac93d5a98bfc2048ac9a24ffb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/acb9b7b9318c321912eebcfdfe1d59ee27deacec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/f0c2e5c218d017ed292b2b128056ffdb524dc67b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149205.56/warc/f92e92638743c4460d03292494121b82dad2c513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/204b3fdf3cb1770db7b431d388ef1e78f01853e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/387a476efa63a8c91e13e0ae42be904e3129f4d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/38ac1bbe36ca2e0d592da84df0cc570b5416b134.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/9fe0ca917faf5f508a3bd0a6a009066e1fa7dbec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/b80183e5dd7d0254f3d25b7c3447840591cf8682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657149819.59/warc/c8de0621e78cef47cf6a52261fd14b8eb622fe07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/26a205d4a45ba5a34fb7e8decc4130d0240aaa09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/341d731f09878aa63fcca6a79ee7e1546b4aa9fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/81156948c9c94706a33c7a4d73bb1d7c8d047e00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/98d30605b7510b3e436557b66456519497878cf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/d728ead8c56e326887c4c5750534a5fb15f3b17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151197.83/warc/e8508713fda95239b7ca06193cdd08b2ca772336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/3192382dff4588072ae8edb8edfd33d692492b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/327404a475b3795376ba73fdcfd0f54744bbdeff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/363c3f436b1edc7e43743c61344e7eab20a34b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/376538e9b65c3da1504838d6b8d7482af025f52c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/8e757fc8fbed294fbb5f1b386a5061cbf0c37479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657151761.87/warc/fb47c744b48b97b3e6aaf020d301ca3078aefc65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/03653a3d1b5f7f56c7498bc2f5c218621cbc3416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/3539c9033b38693e9bbc606471c2ca80cfecaa78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/70365d64c12a86876e584a47b1acdb91fe88800d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/9ade0665e79d72e1666fb27cdab8aff27eaa5343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/ae9c40f9e93a833271b0053459791a9090d210bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657154789.95/warc/f8e03b0effe51b7055c86144c527cc78bb0f6d98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/02efbb551bc06ddd6fe58650872415def6ec9327.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/196e609a54f38ed5c1d3dcbb771ba83795b9ae58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/490af4102be833941d76de380967c7fd0f58bcc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/89df5d3105287a74dac415898f4abf1ea7f3890e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/c9b3d80e87301bf80f9cd09bd5723d51ce0e1b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657155816.86/warc/cd60ebbb0e9f62f70ec500a259f1b82da32e5e58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/0a65309f83b362f893325030f5c15ce747c54448.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/219593e9c680bc6b7c88b754042a794354101fa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/4aac989c7d3b7b3ed6508fe4ff5503626c972541.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/8659245c3a35a4982bd18f5063f98809c339548f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/b6bb04817256ed99e206ea787e34847ae6f63ff2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657163613.94/warc/d630b3f5d7d228c4a2d64cd9ca99cf710b142fdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/06c06085c685321e867f13a52de503f1a55328b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/2c26d12581e858b0de4f9f5687a4acab772e2f6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/3717c2ed3914061e8cd3eca26e236ddf5d93427c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/ab1c26c08f3ce4bef9bed093ae954b5846178538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/d9e2a47f5f98f2372ec4288c661ba8fd3c369d17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657167808.91/warc/e1a7c3509c3d97da0a711647f7b3dafbaa81e11a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/0693325e8f6b8a3b6fdbc7658477bdca3ffb5893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/3896e1fd24344174e6f74504cfcfeab19bdeba1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/3e259abc2a1e33e7e09a1a23ec83c89f5ff9e56a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/80eb1eee8b2898f675f1d9a1ad619510c5c53396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/8bc6db6a6e5ea3e1bbfd525794d804a62ab5c281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657169226.65/warc/cc0ec35718015540a022e3d3c7788b41f0e95de4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/01834830d1da75657d40ed626ccc7bd2cee6c99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/0f7de9014440299a427eaca7a50599da03a4a9c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/2d4c9f445d38c86b2decd07b5c6f3e236be95e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/917822b2a36aed4c50e2cd74bd2b296824bb8084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/c27ad40da4ed0de5b73c3b11210745bc5ea84d23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657170639.97/warc/fc69eeea16805abdac8d214c65aceafa26fa937a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/20e569a269bf6065c5c735771c5314cb11c2018d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/a1ca3f1f49b9c75a1175240f8cfbe88913c8acaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/ca4a16cf64d68596a1cbc6e2f6906a77cd817a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/ccfb83ef0ce986450ecb46eeb1909364fed9f5ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/eb5e67e623fa02b1b2c185a34e928e1af521d1cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657172545.84/warc/f007c22d829493786b17cc09c02649908fe269f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/12fb8aec1a24afc73d9b7eb1a4f5c40c81fdf552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/9126c8a8c4d4a037ce1e8a831f4afb913a18bb66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/96a541e97969cc8bb11fa88e6136f97dd546711c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/aa5b36b033e5c001835ccc261cee8035700fb932.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/b22213bcbc323200e5d8ccf924badcf6fa7de3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657176116.96/warc/b28ded9db2684023112fddca85555b6ead7f650f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/0c69370687f7184860ae00f8a32c75b1680c3607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/6b537cf4efcc2a94495aab6d626ada90dee39a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/946d490253f11f1ed18a9539ed5089776d3d1479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/a054b8b38b48451ee12e38b8792d501e69468ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/aa5a96d80f11f8bbb48d340c193cb7600b2b1f0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/1593657181335.85/warc/ec440e90a29d65ea0248e8aaf7cf5443b42b41ce.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-29 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2020-29/mix/input" + output: "/tmp/cccc-CC-MAIN-2020-29/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2020-34.yaml b/configs/cccc/mixer/CC-MAIN-2020-34.yaml new file mode 100644 index 00000000..45318c75 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2020-34.yaml @@ -0,0 +1,714 @@ + +streams: + - name: cccc-CC-MAIN-2020-34 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/162e6d572290e304272284a46006a052329fce1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/5219c307e7ef056acfe8d6799271c1ffa6f30650.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/7be04bf74eac48bbf36cee8fe95ac26df18cdf43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/84076dbbac56ddf4e5cd6d5795e5c67305d3326a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/9bc9bcda4f3a39c885da034560f4f14a2d7b854e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735792.85/warc/9e209312df6706b188fa19d377976654d24eb6da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/1821d9edc6de43d97a97d9f9a68bb38b6cdad211.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/2a9e379ca0e7d0cb28b937fc485f231af351c3e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/a09bccb09cfbaabe70992dcc64e6e565fb2e13d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/aae4690b96749c4c3137444218b8cfb6cc2a644e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/b31b969bb3ddb77e68f1be767898d4c5559c5855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735810.18/warc/b3e393eab7f6b5a4dc8ed6b3574d2b9c7c50a3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/16ef631b5f5ca280e0aa3009326b0eb7040c17b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/2408b85e74ec5761637dd86eb91f231b1d38a6c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/8a25a4565f751b2420010f156140c15111b89d8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/989d3a46dc7367703a20e66883e573fd7056346c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/e1ee31564c0c8232b4c924ee5c6fa42753486143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735812.88/warc/f50823ad9c0c9b2f916bb3f54ed33aed69f4f521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/00293fa08d5aa392bc5969325cab305da523e213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/58309e13fc84d3d9dd979fa9048d42a1fda4de1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/83b59028dc78008ee72a3d4fb0b51f2eb03ac655.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/c309f844bf97b73a25bf508c198f8f9a49b007a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/cf70f792549239a44cf1709322b1c005556e3345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735823.29/warc/e11817ce4a80d578b9c05cfac3a862c7143fc789.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/1ff4d2b209cd5c97d6579413b8c15f6518f6a89f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/531ea0994a7835bbd7a611a29eabc5e651b1a7d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/947d0dd55f03d80fbe224191f5f8bed016e6bb0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/a642f2a69ae5e181688bdffc8c2c6cda5d62d1a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/af0459c027bae11c488449075d87f07805c34afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735833.83/warc/f7e9eae79c2e45d262279a9f33ef74d546891828.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/3081910f622048134d3362ed9c1d3ea6e4ee247a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/5202d370b2b8b8e87db47372fddb0f9a48775d81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/6469600523684f5c22cef0930d50702955703d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/697a51fc6aaf824bf65adffec352d925254b8924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/c5ccd13483a47f67e67b9ee2aaff2a755b383976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735836.89/warc/f03a7dfd48850203f337456aafb2b36515ddde65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/50255944d5aac9a9debd908d24db7b21fa0b5eb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/682bd7613d8ff1369ca8bec996d21923e84aa97d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/77b73bdb1ee64060c22997225d67208adf6fc3a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/882207b501a1cb9890610533772a442805d41586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/bcf66b934b299d3e69252ed5f76bc3bd837f6f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735851.15/warc/f71198aaef5055c5e7ec5a348c70bd33dbcd9de5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/4ca42db28151e0256c1518d7e4bb145c8d89c3a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/9ae1406ae0ca8a1a86e0623cdee1b36e9f57527d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/cab8c112f76ce2e413fdadcbf7659be456878ccf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/dd302e11a12e04ee8bdd9328314596a425370cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/f41e034d780f426ba6b7712661b36e08e4778fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735860.28/warc/f6fd78610adb36239603241193c2c989d1199d3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/01f20081cff1623ed8666867e1b180ccda116be1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/3f1cad22cec38ff95b83875a77bec88d30b06ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/46dff7ff33530b3efa30cfe3ba790900b959a72f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/5b60c8bdb9052ff7a4495e015973a167f3f5b7a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/5e92e258a30fecf950ad7612d572aa4fb111760a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.23/warc/c55d6ca97dea1603010c95eb274622883e841d9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/55035efde2ba7236a25ff4df8af8a9ccd1b38547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/5987332d36c27808d860c4969843f7d4eca812df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/5aed2bca0b2542d6b3493e2cfbb7fc4937849c47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/6045c15caaad5cc2bf8be7a2325cf2fbefe6f376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/6e6c1c3759fff66498bdbe0683a0a7cd0e297e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.93/warc/effde5f5e9e84a34b48e28b5175fd7b34020cf2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/5c47628953b351ed03f846952b6f954c7aec37a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/b5f660686b95fb4ff70cbc945fa6b51b193788a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/cb2ceedf76499577985244eaee5842776604d473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/dd5cbe7d66c073d40f12d6f5ba8b413563aa01bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/e61357ae52d8bfce3bc957d4d81c50e707d4b288.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735867.94/warc/fd5e219ad9489d806259c0d780e3cfcd8ba9ac2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/69290b6b00ec109ccf5a31bc16b047c9699d427e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/9cc2e2be80ef2ec26caafbddbba6b0848e949e9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/a8cbf1b1b68b55918e83d96caedec5152de55cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/bef35b0d99da6089435d48ce7e81ce18fee88fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/d630ad1b26768c8753dc1c5c2f0bf1b15e17a866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735881.90/warc/d75b960b4e5f7dab0298285a19f60fa899332f83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/3c6c2680ae621acb0c1b48d38b6ba7adeb27b967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/4aba3bf2f670d6b170277361f96658eac9966142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/510b2860c6332ae5e29e084e5f9180ecc971a395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/609a79167ed786ffdb9d572838bf7f57613d5df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/6a18184c6d585d6bebb82daf0c98a92b3b2f3392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735882.86/warc/b454979a1e61397f4461a9ec42a510bbaa78573d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/24250468212b50a48d04b0a561a4cf6b9bd5ec2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/4bd950eed164f24124190ec4b30cb0ec4844cf9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/5f54000ce9de077188e6f1a9a1df04f1295295c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/6979e2151f3b612568dee812a71d2599aadd518f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/811ec0c34faf9f10500a9edf9e3a08c51def9c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735885.72/warc/e10733428d091f2a62864cf702beada6cad626c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/030cfc24fcd9be2835c26bb331c8f5e56a13973e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/33413d673a4b6edd81cb38f6fe7341ede4fdb483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/38761b04fd9ecb6f82eeef5d2801d75f172fffd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/675949e1fc9fe505eb388e4314cf133ea2aa5d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/eba547058a6f773754986ce0146fb9f1ee3f0906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735906.77/warc/f459f8977c6e4d2829d08262cbbdb3a9fc2200d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/3608547340f10e415fbb1b12d0251ebed3c54b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/3a261cc7308109aac0c7513ea733ac70a9b16692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/5d01e825c967c62276db046d2b9e6c6dda3f3f1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/b9b8000261b554283f212641987fc88b7e8d936c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/d12d42a2afad54981b74cb1e531b02e12f0db161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735909.19/warc/d521c453958cdb81d6f76e6f7893a29740e33e03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/267f1e3ff94f2de4d9756259b64fa5ad758d54cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/7c6e8315dc4cab3e673225598ae27e59435c4037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/93cc0b9d4e5ef25b69426261b71695655bc867ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/ca8e220f2a818fa31c8e61848fc55d9c83e9f99e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/f316a76eaf1305d458cc962dc6ecc4717ba8e5a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735916.91/warc/f3c0cf50db0748fad94384b8cbb4333108bf7660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/3ce836d91f137c0a216c7993df8b6d389b69d2e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/4f57d608fdc63b18923ec5252e8e3a671c5980e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/672ddba6d0130dddca9c020ee4c03337d3a393cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/a0505b1188716e28598f60a7ae77d16657963a6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/bc09ce1f41d265baeb35f560280b0a82353cac3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735939.26/warc/e447204745c890d7d677a2144d4768c90438952d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/0a94bbea22ea92ea8fc69a2cbd063a144c4f7c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/3921fcff0dbad3f2b6cf77f79aa14f2719d9cd22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/3c4741302b1cff99b3c02d9cbb2bf702d72e1ecd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/465b7a1263d19b2469a29d394d262ada1629b590.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/63916c3084678ba54ecacbe3eca8c9a4dfbdb028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735958.84/warc/dc5a49e7444eea25da8471024189592ec26e3be7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/30c5d71cd6dc1492d0b6d1e1b6a8de7c60baa67e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/6908060271a18abedcfaa430e9376bccc6e2f982.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/849e8804d38dbe3ba902b3d520977641a68bc1be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/89956dac04a965061e772564cfa2356ee780c830.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/95278843253e7207175087dd8c3a705282e9d15d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735963.64/warc/feeb50c13257f71906e08b583cc2237be6c2bfc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/00a169736f8e31147b55d90fa855785e15ae977c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/02953b4b9242fdf0b837b54c6129d5b12ada6dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/ab04ed47e5a7111fc79c4936012792107f60477c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/ab913c4cee3d04d70e7d244a41b1b3b3d60f2cb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/e4368f7211886b0b0538599b2bca91cf0fad7d25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735964.82/warc/fdbe8e841b60f2133a74e0fa505e2be634159f4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/1f12bff5bdd696c47ff4ce284bdfb72fe41481fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/5f6f7ab4d2f81064a47f0f901f36d866be41ae98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/6c5b600b0060b2e0694a5bc2b4350dd044f99727.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/a0edcc0dad8befbac0d22a2f598a4606750183b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/b4bc81f9a8fc1d728bd92118d7c065831db60241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735989.10/warc/d3edf1d337ccc979283e61118dfe0af27739c5cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/06b635cd0347fd76fdfc74fb13215e217a444b59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/2ed6b3cc2be9ee1ff07a33695e9258793f7bed7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/3599c42b0ff82471f6e25fbd1d032b6ac133b4b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/4712db56eac10a7b38295dfced09d05fdaca8d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/777361bb59a53d4a095e18ad60e10b657d193ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439735990.92/warc/eb46e318d386a207a9c7d6dc1cae1906878ec851.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/0faa04e5aa34ac21957af1063c46114978ae9b11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/23c1bd3483f5b3cfa01bd11bb1490b4b7790958c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/480c101f4a3b038d92129764e5e1352aaf58b9f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/9fe12848322baaab48823396cf529ba6b8288d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/bd58c43ef8d8cbb4039dd0e1f693de97a0add737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736057.87/warc/da975f07967b5e7214d1f46cb20a472420c25100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/12de8469ffe5b601ea22f8b1868b11da94b493b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/17f37f9bb7c8316fb48f9d07b96b13bfb8e70aa3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/7d04107a95f060716ac2b4938fd7252f56bcfec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/833669b900b7d9ffeffb197037dbe4b057a6bb3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/a22872568c4dd0ebf064e34b4919ebd4e74fc74a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736883.40/warc/fa820c6d9ecf5371b61325bc2453026ae121f02e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/49a43b536e7b4f38368db3e246a78bb9271a5b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/5d5b17eef4e7e258ee8d129eaee90e15fe2c5b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/91e6d9dfe9a90864c8bc2ee12d677a77f264b046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/b9e09e62cdcaee2995e27aa320bc3bba69a8280a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/c5e7bc7ee84693c0a227fbc5f7b807fa1e7140c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736902.24/warc/cf48142730248248c6bfe2b807fa419ba2cd8467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/18258bb1030cdc9ce310ce9c54d5f568e8184456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/581df66f8131da327bfd6999268e1c0f1ee2157f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/699dcc8a9dd830f66b9fdb8b05baec453c3fa0b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/81b520dee4ec07790894e3ef04afe2b5b7aefc54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/b4d00fcbdf622b70bbe48aa6efa231c2a0a498d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736962.52/warc/ee4bd4666d6088491ff2bed941439eda200ea949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/1e18903e9c1d65e2d56b7c5c5f09a37a9956a577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/32896fd9769839bfbf2273b2d59c8632a53a5c79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/33cb1dab3b0a0c27a8b7e0f01c9dc59478f2ead4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/926f687b16ce27d82bc720ed86ca09ddcd2b429d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/e16e5c384855b4affd50dcb0ad228c8d4d0e0d02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439736972.79/warc/f000128b4bd900ac587c3d24283fdca90daf6574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/6085af0ad13321883c60c755b45904cffbf5efe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/71d6404e2a74bb68ad83ecccbc88a89ea1eea947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/7d764a3655bb6a4fbe1c9d28785599ad1ab49e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/b5b14007917abb90d65490a589d065e1d2e3ebba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/c616e1cd7f760a6c8a4ef917de867bcc88557701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737019.4/warc/f7bb8e5d0ee5ac710b6f01898b8246b634bfdbfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/0468398a1ac2bcbd651da14e536db9da1867a737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/4396974dffd36e9cc9eacbcaac880eb580fb8c53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/525c9a2f3bd02e44c8668101306bac91a285bbaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/807c294320340a3bbc0260c743bfdb71aed06b39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/94ccfb994b71fdd94e2632293321055a49333ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737039.58/warc/df9e9ad76f427542321299a155f557a4c5820ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/3d8657a1d3bcff35b530f9444f431e8354ee8304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/976754ebf13532efff6ba3cac13e74cf2fccd957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/bc72e1952dab31a6d47ffc32840599a074687baf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/c0bfd5bfe2e6df023d8cc4b58d563127cb8365de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/d6c63d6e6e3df1032d2345d4bad1dfa19ed0cc3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737050.56/warc/f93c174a8bf0f61c36bca87e2e94ba8f85d662cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/093b564099130e22aae935befbba493ad5a2a611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/40c27be24b6486f10e343312ce0ed8224bf4b56a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/47598336b2e82842c80c67c694d19775d7ec7fba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/7bb2378863d87f4b18c4ee6c9c26ac5bbd046f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/9962714e94c01d87f20d78857becabbbad8ea66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737152.0/warc/ea9b0728d1a360eb7ef6a65808dbde7ef9bd9fe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/0f80ea5ae845ab0320d17d447d2ef61b939ccf34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/31f943c60b509fd8d3d506a9c833530d6b466856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/4819502414831d28e671a8f067ab77c1a092756a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/ab508b6b33de79ddece8035e4653852c0a879853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/cba5175d1ce250e5e9a0e6dfd2165ab721c88d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737168.97/warc/d071d95007feca6f48341300747ba48761d9a53f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/1a7f4babe8fd18715c8a2bba349abc7cb786beef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/4315e5f4074d6f0cf17ec68a64908645c718bf6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/736808ac20f27f017a7b6ad4cc7f0847bc83d3cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/9360fb198e786098ade4884955024050f3e1c708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/a0e6dc860eb4618ccab5273bdbcd5518388d2305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737172.50/warc/b6b735954d8bd4fdb18473701f5a3e6e4b3d2079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/1a303e084a7148c71e86c5b0de7731537db5bdb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/1fb0a3e5496124ab30b63ebfd3b6758905823df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/771c9e6d611d84a23b6e644cdd102ed28a99114e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/d7b122f987cc3ed1a86c576a0af89f2f003b5e48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/da8ccd04d2a89afd1926b42244c170c978d38e19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737178.6/warc/edebbe5bca50babb2d35ab90c805e63654547673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/51e77119a11232d1193551c28df0ef4e6cb36254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/78e7c3cee5845eea21768fb004bce927c6694777.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/7ed2d264b18d829cd688777f0f02a391950f4107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/ba9e522c850830c3c892b93272147117949a35c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/bb4bdc754256aa33f413f7add2755ddabd3f3b09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737204.32/warc/ee16ce4a53d99281c49f1e30bfed45120a9cfeff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/1cefd1fdc0f6a6b09531f144b9e40700da62ead5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/49bf275c44c7acf1e64a1d11cddc1010be04a1a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/7ab9f96cccc16c1996b0e91018f9e3f54476741b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/900ca2cc3864761917ef502ee847b98722e53729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/916e11750e6cc1aa88a5ab3ba28665fe59841367.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737206.16/warc/9ac8c81032559f4dcad59456824de9a3d2102901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/5ad21ae6bc499a27f8580702017525ccf270dede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/652edac14ace3355b66ace029b3bad2c200b6edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/76703937075fc344aee87453d21697f1f123c776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/b44c9515da94b30daca24b306eb8bfada9d70cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/dddc761034e111c1be4e432ac149cc967665eff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737225.57/warc/fc94e3fe0b520f59ad74a10e48a9f7d141a80e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/5fa88c498fc368da74130be1f1798c6c597b5d22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/86aedc55b8e98222b5a9e299071d708d56cf2994.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/8aa6e47e4d151ae134025de161e723412269815f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/c580eca8ff5b8c4bdc7a01dfa3c1cb24b3bf89eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/e1bbd4848c7b0ba7ef786d05498f4501a5c68889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737233.51/warc/fbb403d0ecf723f000823159f50532643eb76e97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/0724ddda879c906f3c4a76644a8f9d4fc8adb109.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/1d376fbe5a3b9a74f2deaf3659d399605236b7aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/74835fa7a513a9d0bab9e2a0eba1728f1e38feba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/dba12f99ee067774e475f8005f18a75b493715cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/e322313c52804fb6f53aa5dee946ee56256250f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737238.53/warc/ffb6be58a15f32b90a4591e5a51107c1f8f31f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/11f990498c79c13cf82f6b110bdddb388e7ff73c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/42d1065868bdef7288f347610522f616b42afe30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/8abe1c19ad5429694f0826dbcb5bbaf34efe9b88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/8f77d69a84619d8f8d7f40a24fd7c2a39ca045c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/c70080fedd4665b75f38ae5fd4a23335dfa8ddf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737289.75/warc/d5c747e670db9fb2e9f79fb4a990f06aa025612c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/5fcb8b4182c461f3883fa56c1c1445f77771a243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/6668ca52a601256677f9b899e55383467ac0f199.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/a535b10972bcaa1d227b726fbc8f5473aadd4fec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/f6ed724a573a89250ce4c2d745b47efc21f99b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/f722c5d9a29a12c3dc3d5b9553b3bc280bbc18d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737319.74/warc/fb47559e6f1da7023e215dc638b43b7189ae4281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/33836ef569455d9b19644aadf1ce3f37ee75cded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/3983f95baf1e8ccf78891b8d63925b49620e4188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/4ab9edc8c1a1f2028b4154d1bbb955bde7ca838f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/5f35c12d6bbf96a180552c6109323a0ca50d4800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/7db1ac24a3282ad413833446a2a9b655423a8344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737645.2/warc/ddfbb35185f4d2326432213c4d56ae15ef2d4588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/14d0a4e4676514678ff62255cf0ac3038fd7dedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/204977c53a591d0219bfb00f979272c2288fca36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/3b28bf40ae1dc519f1f055df4a92147f152432a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/5c80932449efa93b73719973ba30dc0cb77c9086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/913dbab83f30b7f904917788aec91336ecaf76cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439737883.59/warc/d29cb2dffaf8a1869c9285519837a2c56faffab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/01e4b610fbf7acbc53b54e2bb8ed78053e560227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/55cd2e285389fdcba58d734db70f88631cd8cb5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/580581b832b4255ae3c7f4947c6c5842cd47d6a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/7c893e9435e4729398a54d448dc23fc60c888841.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/b5ef28835482f6d62fceb54c57f1cfb00a3e0801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738015.38/warc/ff0f7f93740295ed27107159fe5668ade1e44fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/53cbc4ecf91f75a04fc5a9531d27b7717fff0e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/5a53916972a8df971c32194848e6bfff116c88ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/73dc20800f4ca6871f9e20f4468133a4c0b02208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/9a68e2908398be21559360eabc4fef8a489ef136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/a9a8472678be06d13e12c4ae261cc8b7152a627b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738351.71/warc/f2b4b2652d6438f1ef0f071b273f5c5adcdba6ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/66e1253b9b37c592247d5d4d1a19388f51838e9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/768de7fa77a26c0e55bc7c96863c701122d398ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/b230fafc5d17dbffc7a1205545f39374ee4a6254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/f305274bcdf701e102ecb053fb8d2270229ab637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/f329a82fa38ec92b065d8604f5162210595f16d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738366.27/warc/f534b6d17df2af5d4f8fc05e66554711b3af06c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/048c721cf3227a832a8fcc0421236beeefff0c9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/102184e76b79ba7afdcd425dbf5c868de2264282.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/350e4d7853f897ae1e45888ccc8977df49923232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/7e65cdf9c38636374f3a980f79f4dd383a261cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/b034da7d9d00fcd70c80ad78239e4d72f01d6463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738380.22/warc/ce8c39b34451f7ea7817eb2f025d0a626af251fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/3a8de441499e82df6cc60d923ca3190c34449c4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/55d1cc27bb11e01f40913f6a51bd9d1a6bd6ba83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/731035126fecec2f7d5606d88e040c94a6dc8e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/ad9867fd44cea5dc1ea51d10488b72dbb781802f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/da7b9688d57f88ed8e5027922abca21a4c7eb6bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738425.43/warc/ee86614624d2b32acd9ad2dbd1b3f6325dd210d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/12abbd7a1b37ebc7052d39606b8c839467621b8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/6b304e2965c4848c31fa460140ab3da20e8d40c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/7506dd3873a7adfd2eed59de869dbfb7583e6c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/78bb15559ec0321b5b32d674b9b05c6a16f23a4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/84e753bf5e6a43276dd9f6edd3ff30b1beaf1943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738523.63/warc/e8055aeb9a160c550ae276770e0f302b3f594e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/15477ddb8e658b6e43e42ad6e22a6cd07ea8e690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/21c17b6260a324e7868afbeba6b5aee6c32ae033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/691ceb94c19965aa7456d8161dcff454e8203861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/d5bc3103a4b88730ed9f307b783a92c9dbab8ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/e854904885af9bdcdc2362a8ae32af5ff0f3d1cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738552.17/warc/ef80da6c4dead789332369623f59b32cdc647876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/273e1778e97e1f4f6d3ae7460606857471c9b67b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/8a31916e0d6130191a6e850fa174f07d6c0b7a3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/8f25f6f4b73aac337c468cf6cd4a4c74ee19e447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/b2cea8d500f49379a7d113168d3bd6f201d8ced9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/fca6730b02e5f5b6b68fe3cb3710f58c239e399d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738555.33/warc/fcbab6ba1247ed72b7f9400e4b2fa3bf537d484b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/1a1135a0d6bb625bef792089f4fe9bce87a9fe0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/8b2c7bc3112639828bfd5e5f99d1007c5406faad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/91e50fb969d103118eca60cbb8686b3026d5ce32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/94fbef1661bbf533d40ade46d17c91e6f497f52a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/9685d58cdfc83462ed62adf42b94d6e79d8ffb32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738562.5/warc/c4957fb18abeeff7cafdb91e8a03d7ed64911812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/054e5a45a28f95afcb37d5e73347bcb3a0420554.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/209313d136bbe5921bba1497fdb99a9be88e5dfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/8567eb1f74c992ddc4904ebfa4197b1d1c0bdca4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/cbb0485fd6dadfc8228a2783ed1346916d08637c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/cd00724261ef353f40049302940d06ad73c5be3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738573.99/warc/e918aba0c028e521ac3e196544136b6afb529f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/2e2247029a033969eddbac4b3b04ef077b9e5c04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/5660318f624f8f7cc153385232707e3a3f1a4918.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/582b6edb43f7049bb1000e5a8be8d0defcf5c4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/7a01b95141efe113a5d86bb7e9e2c138af8e481b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/9eb85181b97485d393be7e20ba410090a6aa97c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738595.30/warc/fc0da1f0c1e9923823e7b1caf2e35218f4fffa96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/46de18be7b027375cf1b80a153df78fa0b8aa308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/8583257544bd8d3de7a4a497cd9fe093d824f4e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/968d3f7f6a272cf555945be3975664d6a8ce2ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/c471160bec1748321e9604daafd6c8fe39411c1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/c7d6b51b7ebe2e04cf0d0f2edd9edb5433c5b8a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738603.37/warc/f0e8908b51bfa30bcff4470ffc7c99cb317eb9bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/0df2f5076b3e6f4be258d37521a7ff3cd77ab604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/1b9d2f7419389b09f766d5fdeab57f5133973b52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/6d7f7145b46c2933caad5b2f87c1aa03d3e4545e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/7543813657dd39ab500035683159aad6fd6b3f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/d6066ad71830c29f8b49d9ad00d5e0842aaf5d15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738609.73/warc/f2ed8d782c8712ecf9268bce634c855dce2402a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/1c56b70ff9fdd324eca0c926820873995d8e4c44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/29ca31799dba8a9612b25e21d6bd1877ed6e2326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/2a38071dd19a456abc6f27d3bf705319ab164b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/3d24d7a3f998c797d68effa9f970059c212165f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/44ff1d3a9edfd185c3004c34883c987dcf3dd010.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738653.47/warc/bcd5d29f8126a0f90f77c02c55cc0fa6b1889cde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/11aa44c5000b8ad6437d47951c330320c016faaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/20b57845e445476efc37d523cc4f118be198e122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/5516042a3f8565267466d6b33420fb693dce05df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/6625b968370c59a79c3b30482a74eb2a02b8a8a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/b833659b83f03e77a216cb1a761ab7362cb9c2ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738674.42/warc/d36b7d3943a7662d6e0135429095c40bde8d9021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/3e25631803a8732343cb440b62c7c44233e8b428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/878f87bc52048ae46bdf3a4fc3ad2018c20f1d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/969a3ddf8e4a2fdb6c19ee127e653fe90fcf7d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/ae11f920836917866871abc7a5db95ebb696dd53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/d9cc3b91e9fd0c8a9c67eeeecd42854435436351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738699.68/warc/e3895e46c7d154529e983be8ee6451d2a94bca9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/0c109ef0c868c196be28bc073664fabe42737bc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/60eaf8e940522c6926f69614f9c7f8084b451c54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/6efbd2cd7543c6a7ca7a8a64428f496173da375a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/d3d15c7f977f145e9604dc192e8fb62e34d67729.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/e1912942707f056576de72785656cbad4e870fba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738723.55/warc/f58d4b14b1b5f0ec7d06187ea06a70e72df29bb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/0fa74fea3adf833559fda896e35a6eb4d408e64c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/346a97a61401f469cf1c5e072ed779fd8933ab24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/6ed1205d0aaaf734f13c6e36159044326b918df3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/70abeff3def95525a901bcb7c00ea6735aef37f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/e262506b3ed67c4602f34eed9018bb85ee90e958.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738727.76/warc/f57197b11a18b187a630b191a125561fd7a90fec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/2d08058041bed0aae0a2e46c684fab725adc36a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/41b10a620a5126edb6ea225357828c631cb37f4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/a897607d7641722b1ec773be25171ddcb6743b13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/ad8779a0dc21b8aedf606e0363bd281236127e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/ef3c84226b252a540dd3f757fef16bbeb7bc50c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738735.44/warc/f63bf0c34de9f6fb0ea9b6a3f1a4e8bd7bc3fe6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/308bdeae63fd1273d401e27cc3fe5669e824fcc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/56d1354243d7de60a56442247e543410e26645cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/73f1275dced8fd3350f26c6a5d164fd1beb38779.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/a7aece9e3d6046938eff9f1282efcb96150dd68b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/e84335001c2b89604f405371812a8cc8723e0a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738746.41/warc/f9be5311a99800a736857fee69368796d6bbafbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/101a79f61692b1c112458e66205c10939829a53b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/22bfe875800203062b866d0de26b3aee8a31cadc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/44c1a68caaf7b87758ea286516db534d4ffd1cd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/7338159bf291d6d7bd775b1b7b61186c0f77b517.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/8351e81292d9c5394b2a08909ca893bc3574ddef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738777.54/warc/b833497397d43f087697ca8751af833a530053ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/1305485e0a78bcd505c9d9552059001a8d65463d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/4708e1b94bfe938e1f84bc408f9476253738f6de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/4be29cd764e5dcbab7e001da2741f44e0dc673e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/582bdbfbd4caaa788d98f178f093dce352d25bad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/7a31715a2bfed9abbd758acf841806b19df4428c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738816.7/warc/f85d83cde50f0958d8ed90fee398e62c06a53753.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/816946d6a6760b6812f4500b2a3820811602f079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/83101920e6f9417d618ebd6e65c98c82b526899a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/b9bc9f3c4c70f50a154a60dd4dd6b6240469d2ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/e2f1f0cb5432128232a45dac6e5595676c7ed9f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/f3b7085168c90ed7106b9dffc4244a0bc9171633.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738819.78/warc/f5499d70047e6828a4311da100a10d415106e9c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/11ac022e3241a030cb3d068abcfd5c36d33722cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/8b72c389ec9478bc7ab6df347c8ef35f2bd6ee99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/c6e665e55e2d5a06101801fd9423a14b98c0f794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/d44575deae55e9e0d48e7e4cca3987457ea995c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/e3d76981b2ace2c194747c98d536fa4e2d5c95f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738855.80/warc/f01e5105b545df31f2ace7ad8b4611f035641759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/0815d728584be93e3fc2f0051ca0e71acd7713fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/0c52d82082ffb4130469221ce024b4f7d4ce0fc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/6460d8141102bea4135c2e67b2544741dbf45c2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/92079a7b99a33a180c75bf3c3495c89550ac365c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/b352e6ce005be05515719dc1a295456784e39e01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738858.45/warc/d8ac79f26e74d7afd54c727e48e9387bd88c31db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/0e26cae1e32eba8f38e9ec090862e4f6f795d869.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/1d1201c7fb9c2f67c8d0e2f3876ba168d98ef5d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/3062a0bc97c63e39932a03ac4cb8aef1bc992d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/945e8e1ea4e8d3f0ce31f9112dacca033bfed5e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/e50eda86dd73d035d2ce6d301e1cad1a31c3b50d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738864.9/warc/f6eac8cacb4bc11ef98e4fef4d2603a359358b9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/0ac298f030c6259af18b2eb4d3358587fc2edd08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/3584eb56a17f918e77a2dd3451d25d43be0aa8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/5ed6b70b594d6da5cf66f9214d9b44d23ac8e900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/7a0841ca101ecbd8095c76dd03034ab59b43b407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/b1c1b426f4b6aea9a694f1af9b5486c22e4975fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738878.11/warc/eb371a2698d003e4973292ebbd188ee5becbcba9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/235f5ee02ad11ad2bc2560fc6e9496358d2f1dfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/2aeb1630ad79d8f013b0383fc7fc96aab7a85d4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/652a88939c52290ca44cd80d54d99a9e21dcb631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/6e82e65de6ceb24f47066a6c7d2763cffa7c7832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/b6aef7fe520f6f2bb589431c4cda0a2b4d143569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738888.13/warc/cd6758a82b809f6470ffa8f91c8240d03bf6de0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/3b02932fb2577a06ce60b783087946f0818e3091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/731bbf4b72b2e581c80c8e9170be76b3b21fb444.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/7390589875bebf717889a8ee51c1d9e2edf3d7b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/74682ada8812ddc22747a63b7829250d0f9bf3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/7b3baf8b75fff86ff6293475d231ec5c8bfb35f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738892.21/warc/d20c8cacc1f3fb94250cc3fb66a8cb30f7f4bccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/1a2cc21e808a22504bb5eea20229ec5e006696cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/3f284a1df1c2e95a6c62628f695b3d3a9ccc8ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/56737ec9a5a4c81a2aa4de1e8e0a3f1980a5d898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/584158db6b31795e4f75474b770f035ba356dbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/680ca6abc979200e8c976ac75fb14573c60ea703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738905.62/warc/9e098aa5bab6c8967abee493bfd393ef0063e15d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/58fd6eb33d2394c5325188ad7f59ffd3febc27ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/786b60e15aaf9b5adecba9d7b0f788b93aaa6ec8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/7b9a6c5b35a078605af7ba59c270dec76221cd5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/a28fcf9b3a1b9155764c96706a137e4705f377dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/d63a163ecf9f9a9dfc399a20516ec4985ed90e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738913.60/warc/f5a2e05b006f7e4ea6b11692b853039f5a51afb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/1d1962b38bd4923d9ae04ba37565c0d9ce926115.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/3d479b9cb5fd76a0daca076fad311753a925c261.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/4583c72fedb7e91275666f78d4ba3741f3074f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/5fde24dd5dfbf12eded9f6fbb17fb4559a3973ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/8713d2fca39d2a6d02b063112da04354bbc5d596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738944.95/warc/9316704063848379834d7eebfb793c5b7ccfbefb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/20d494f65fd4b565c50cd7ab2c7e79aa6ffeb79f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/2fa9f291e2eb2d2c5c72792b52a05e49d9439b5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/342a9ad836093078f90859d6f3dc94e9696bfc17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/93cb6ca81e0826580b567fd026d92e0d3173e18a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/d87987d902b04b5e71053da5708a2248f5ad9c73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.31/warc/e098c0b417b3c544718e30bf200abd7faf909b01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/05d0acb8d1cb98e408cbd23c9f873fa478dd71f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/6b74d57926c66b8d52f7aadb8eb08ecbfa8e95fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/78855661f25268378707879e8752c1bc5233845e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/821e5e166455639f4c5f2d464885b5aa007d6b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/9ea901d9076d8971c0274001a18272c5628991d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738950.61/warc/f10df8ed4461c27498e2a2ad599758e79c767b8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/090108a7be1e2885cb16c9a787ebffdf301530a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/119767b98238c715abb2c6e793ff8237f320b9ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/473946e8bdb60b26067fcdb4bec92eb2568505af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/4dcd16826f605ed61e0f01d1e958bead359bc9b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/8739e626c9e3c6311d35df48896e7ce9d08ab83e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738960.69/warc/aa9cec3ca14baff2a96d802d0acd743f78629bb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/2b818472250d9355189ac8aa4d2b76973a36930f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/6593ba19c82a97953b08aaaecc79feaeb521a580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/709de75c126f7787fedcd7eba431061b94c68ea5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/72f08b6f7970bb44f79d359d585bbd97f92e3a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/ca3e5afec9f965a0627edc804b2aea2c6eae85db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738964.20/warc/f17430a637131250ed07ebba54e1aaeedb97cda5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/4455b7233be103b8d4f510e3dfbe162433f2622d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/625eb4ff00eba5de4bf815ae9bc761708630fbcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/748df3024d3aad093a934ba067a8dc5352e35006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/a7e548716088149a8a3d0d565bc1c204c40e959d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/b8cf0683a5b0524aaa4647ae1f812f9018a7085a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439738982.70/warc/c47ad8fac5b5dde0b3da33f1001bb45642996057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/5e8423ac4e0909f404361b5fbc0fdaf57bdacb87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/63eb75041f15ee339a9e6a124c209dbc59689b9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/6d4afc5771fc837bdfe70ffa9640e01a0cce5486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/84b8843c9f78cf45b0935edd09465991949c4324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/de0735c8260dda125cc196e892622cf93803ad53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739046.14/warc/fcd0a95dc6b931abab81682af1d30871cd3f31a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/339d55b90cd6b9cea08e2621b5b2a02730d471c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/601a4669eae79df75e4b7235423921474aee3cdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/a200ec380456b018ff943a6e0393d8bc4caf20fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/e69e2dee188efaed1788160fbb6f8f5d91fc9a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/e932f712baa6632f1663af0df705d4dfaaad125e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739048.46/warc/f7538207b3e68c198d47c81fc9e9af747da01546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/054498c67a141fdd0cfe7ac2beb582739970147c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/4dc125234f6630257d82bf337a3ccae5d2b716b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/6d5a8f8d47caf7fd8ae51c67389e8fc5f1370b06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/842634b3af003ad6a2db5f38395565d28047caf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/aab4979a8b9d4c1bfe649d627791d96739ff8a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739073.12/warc/ec9d49ca98fbc1e4d34b13ba9dc024b7b0f06c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/145dec8ded2a0072b2c0a2499561d0098f36ed33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/77fd1bda12d8407987076994244f557b5b1637f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/841b2ccc67ae589ae46e1594c6ad81e453afeaa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/85df8ee4e11ffbc15b2a222426d68af1f0566edf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/c804c0b4080b261eb1faf49940ec0afa7e348dd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739104.67/warc/f70ea9f14e15403b3a8c1eb15f636badc8001766.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/3fd5ce16c0a7800bccb4414e8fd970c04a360419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/922d6635a1d560ebb9fb94cfba19779556d89e65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/95fe59187da23f1dd6f6e8191b2944d3cbc268c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/b08b58aa18fb052bc30967a10d178c496504c930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/da1bee4a11219bfcc151d587fad6f62a76d1f300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739134.49/warc/e091c21aa2356d0cfaeec6d656d656b61735ff95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/16d18045203d6dcd69fc3f375e22ed1522a46bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/29706c214e07b783addbcad79f37d814955e6d0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/5562f9818e51d8293ce14295b0b6e84bb67a6b69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/7f2b300ff07b58bbc625a872b19713c36ee5661e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/ae9cac7425a30974efecb4d75db20f846f44d3b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739177.25/warc/b2133ca442d6a746801f8142ad1637aaea3548c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/1ee59665fbcec030b8ced9f7d76a8b169b87e13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/228f86a40aa167262f717fc52870811f25bb37b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/23f28da77fd0def3901489d7b8ab05e1c4a8496b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/8ad089e72780968653be94f184903e5d3435c74e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/ac7d6d10f635aeb318b1a606bdca6a264dd347f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739182.35/warc/fa700483c677994b08cced59d750fa85ee039b6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/141b50d0c94587fb9e505c05ea2e11b87301b4c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/521508711443b52867f5bf3a1b56a7406f186737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/80b9ec50913738cecb56a969e8278a142eddd368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/89fa417b841fc15f8e475c5427050ff0dda2f32c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/b5d794897fad22d88ce361c13deedd3c18a68107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739211.34/warc/eee955183618feddbfa1e39ef68f81df7c44ce9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/33373bfea5e4bfe5c1e0051cd54295283cb743f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/3916aa353ef539d7dc05415759479dbaf319d44c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/922ed27b21da54591b1cbdabb07ecedf582a3a5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/ac0de1910ccf4f02543a9ffbed84c10eb9c866dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/d069a1bcd26589614167abb5bb503e97dd66c89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739328.66/warc/d8ae2626eee7df39e94ffc92f93ce00b9aad6aa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/7043a15bfa4775a7e9de55d1dbcedb3a10b2fde9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/8e6bbad084fbbfe49f0cb6b3bc117208baf2e9d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/989848ec154c1d28d022d670b55061445b34d400.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/9b135e23bab7befb37687e7c50919f3359f81d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/da4520cf065dc394ba44961a0855fe70e36495ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739347.81/warc/e16542c4a5050c84fc0ca6a6f0cce0a365b5d0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/05528d239c02fc7ab9cb95937c7621ea35e34092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/6df5d6efcc65c89b27d2d7152b1a9ab753243523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/7f7f54b1a88c177872b5d4d4f4962aaeba0ff7d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/9eb2bb8a719b85f3c958f255c4d436e486f51e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/dafed5900f9c5ad14a11c16620087464839f7103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439739370.8/warc/eee96c4d8565f306761cb588e332d4dae64d09d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/08bd1e84d25becb879a9ac6645d464a4923688f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/3b08c3ad236516e8e7d58254b96cea52276ec107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/6bdee85ab48febb71ee0d7fbe4a48ca048184d7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/874d64464c02c74176cc6a00b4fd9b33b1bc2666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/8e8af5918b853d8a6d90c68b780b66d00f92a0f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740343.48/warc/c5c14d8efe25a8b26dc646d63178ea0ef115ade7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/08fa86ca8de381cc542c5b79f74422c84ce18f92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/2f1c70ca7b1159a4d2422e7b4af5e3ae731b288d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/ae50105426e9a86826c0a862cd3a2c7e3e43b804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/afdf6275e1df112683b59d13a2f0d1db4d582e36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/b81717d5a1c9ba8f941c38569d307164d8304418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740423.36/warc/c98bb924cab392c5a009a2420b9998a557d22482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/76dc829f0cd8ba27d82e842416d7721f20213ea7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/819e38436e9724e585dcd437efe67f40c9e25f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/92a7f9a88e4c72a627a6cda68f177f473f07aa5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/aa4554fca867267e5d73f5ba34d4fba54c6a1f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/bb0bc51551925600535c60a484356bfdf08aa7e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740679.96/warc/e03faf1ca3bc93d2707dbcd60c75bafbd3cd187c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/074a0672b93b3f4653ef26ed94d3004fc01d07e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/65e7f18a2d1c451f9dcb381919d3c169f21d0b0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/aaf2c1fcff40e739a7b64242fbd100608fc78d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/affe18bdf95be59d2c79c6fdcb4d70216a87cbff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/d17d71fe35891d9553e86da5758511062f818522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740733.1/warc/e9b8b89f3003fad98f7184c5430d59ba84c9b2a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/79fa199915ad01926b290f7c6ea7afb201541ae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/7cf186bd66fe01d86302dc34ed7ccb3d541ef5d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/9a785d28ca3e5ff8c7e2d4eb8a8061683f416e6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/ad5e08dce73215c2bfc07e904e08702074f409e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/c7833904d3a592bef75a4583832101969fcc89a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740838.3/warc/f558ae316ef1f49c6e757afea769a0b968ced0e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/22e0a74e112f87e54755e075022d52452b560d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/373ec66986b75aae95b00b2a0d09d40ee1c24234.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/46db730b2490f45c937feb25fe5985a0bc5daa6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/4e6da717bef1c1e6e520d52af5463c0affbcb957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/644304b4304b796060c02c44e4ec840f0d63b413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740848.39/warc/7d6143e68ffd18e5f09bf25ea4e9cacf93f8407f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/19830ab685f3fba961788bc2aaff7711b45a3bed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/293bf4602c97fc09224d773e54a0d1f39b14e796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/44118d1f939ff1f9b6c1b9e34aa578a6eb373797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/52490b1ca5c684cd018cf36f6f6a5ef2070361db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/a48404adb84b295e3c83edcb59cc0903d0dd4b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439740929.65/warc/f11d87fac677adccb0a4957fa95eccc8df269584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/0399387afbf337c2ce38c15d4f841d859600f92a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/1f038041efb12d687c8ed05ff9266aad86c4c71b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/29ae80c449cb7750feb2bac54deb540035c18d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/4934873a27d8d3aa7130628a9105d791ac565615.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/4f461ead7a7381c40c9059b714a62a9a7c41b5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/1596439741154.98/warc/f3789c7fe0a9c656723e3aec6643059ebb874b22.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2020-34 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2020-34/mix/input" + output: "/tmp/cccc-CC-MAIN-2020-34/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2021-17.yaml b/configs/cccc/mixer/CC-MAIN-2021-17.yaml new file mode 100644 index 00000000..b3036323 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2021-17.yaml @@ -0,0 +1,349 @@ + +streams: + - name: cccc-CC-MAIN-2021-17 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/0f033336b6fc23696baae74b8b083b0e3a13c8fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056325.1/warc/0dedcf45b255b94a9fcf76f616107a16cfd22da7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056325.1/warc/17b4d198b30d37e7d6402adfb362f479495186f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056325.1/warc/4677324dfef4453db078c95dfcfdd3b62f6ddbec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056325.1/warc/bd43e7175db8f80e22ec34f7c42d20dc152b2e51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056869.3/warc/0998a555969bde86e91ae565edab949b9823f878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056869.3/warc/0ad813539a52dd19fce1f180ae738b1d20d01b0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056869.3/warc/2738325a432dc034da24193a9a7ebb6c7fe079db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056869.3/warc/59e10d0ad4656d34b57a26494e301e51b114be1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038056869.3/warc/b5eb1a478893818e6fa892c0b26a182229b0b0eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/01996b2fc1b09865cb97ad1164890e485a747a94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/33cb33fbbf878e6645b07be6962e1d16fdb1a797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/4fc0895593ab39526c9b87ad4128366de07ab033.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/7c38d532579f218f1d06d8cb6b1de3bf31ccb89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/87b55d7ad74dfd62646f06cff6d9ce7e0142e6bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057142.4/warc/f169f85f4174b3fe312c4c1a0d6f08d052883a83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057476.6/warc/10b1b39456bef1fd6f5496af94fcdab0a57b62a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057476.6/warc/3eb50431c42ad1978862868b1be8b8df4ac44328.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057476.6/warc/4c558e5bc39f442bdd057b089988b8f17a0a1580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038057476.6/warc/8da2c02aaa03d53d255068c1b96d6e445ae613ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/12ea9fc4e88a4e4c469f20169ed30fbe81f7e213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/1f1288359a9d8bce4f9e1634189fa293c07ae6d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/5f260ebd1b4a720cdbc48171bc937c64afbdc6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/9b0d3c360b030aa544ac6998999088e368538db2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/a3fc4a871639e1494e93e6a29c0637f1df806a49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038059348.9/warc/bf326d478578ca39fd1975fa964d130b4e378232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060603.10/warc/59587a0559a07cb01542593ea990f594f08d56a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060603.10/warc/afd0a94a20553424461f05ec646e14d8df3261d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060603.10/warc/bbb3521a0c65a072b63e04e0069871debc004b18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060603.10/warc/bcddba0a9a20182cdc37f389f9c91a7ce92c0291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060603.10/warc/cca66f26e681bbf22c33a4173057501536a0d68a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060927.2/warc/159a325a8c45dc7599c618e87810388c677dcb54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060927.2/warc/78a8e0c2bf754bd1037cc364956bf8f0a87c5163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060927.2/warc/7fed51bc6cae0c71e61602663d6caa84a472e0ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060927.2/warc/9ff5bf437ade8e694208cac487616a416ebda481.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038060927.2/warc/e356cd82c38e18b60df8dab502fd4d4cf8392f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061562.11/warc/a39e84984e477c823830f51f6a2619dd7402df13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061562.11/warc/d76e09a753867d8ec1e3dcad1a2cfff52ecbdd8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061562.11/warc/e3940dfbc72dc358c98aa22ac0c5796438e2a8a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061820.19/warc/770c6610e54098ed36e0d50686b941e88acb7822.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061820.19/warc/7870a387529f2a73d1db6559d5666dc0b1e75193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038061820.19/warc/b33104fe1bc4973dc142d819f061f7f501375b3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038062492.5/warc/0b1cdcbae5ac0ea43a13049bbc0fa2c418bfd0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038062492.5/warc/71a96ebf61568b7c4bb24d5ef8dd5bded55861df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038062492.5/warc/7f92316c30a5efc091c171ae9195378046f516fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038062492.5/warc/b92f2d0e235db263ce00d731c42e34589e917b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038062492.5/warc/f79e82f7e1fad2139171a7345d72db3f3c7584a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064520.8/warc/2c99b8c9cc0a914222be09e6fc43cf09f49c291c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064520.8/warc/3230904facdfb1ffa5bc9c95c973e476d6ec0d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064520.8/warc/5ee9646d620d694382ed704361c3d57a22595a58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064520.8/warc/7f7b4b4953d42fd2e38e34d101dfec29983b3904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064520.8/warc/e05650d9f83e9d6f0a303ee281ae265695ea34ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064898.14/warc/3dded0fd6da541dec10b4c26aee5057eb2419e3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064898.14/warc/55585c2b8d63172cd0e01d9537f30e79a3489cdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064898.14/warc/ae53b45c6efa31bd3e69cece2bccae9571acb824.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038064898.14/warc/d3e1133bb4fba4459bc57ce80a0ae4813d7830e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065492.15/warc/1952bfde997257b61472aa071eed52af6d43698d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065492.15/warc/463ac94664ebdd19a955280080b26ce2a5ab8269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065492.15/warc/5cdc16aef6ebc6a89aaf4909db0b77103fcf59fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065492.15/warc/94a8d6af021ba2a73a891585fe4b5df6349d552f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065492.15/warc/b035d2f3b21e952fcacb8e8709fea5c3d4836fc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065903.7/warc/6dd280d5faa231330a8be89fc34b5902e550ac22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065903.7/warc/6ed357ecbe0768304724bf800c262376c5fd3cbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065903.7/warc/ad381203d8395ff03c96adfa3a869dcef4ac432e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065903.7/warc/c8407b66708b32340a8933a1682f1f1d9e29886d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038065903.7/warc/db665ec69cb44d5cb6249159c3a4866c1208dbbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066568.16/warc/2d3ca5bb42c5e235b47532b2728e135584df9344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066568.16/warc/8b5de058b8569fab2db23792afcd8794248d8127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066568.16/warc/d87792c3f577b7d2b984d9f9543d0dda11a181b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066613.21/warc/0c780f20bdbfa1901dae02f97a9e3d282fcc362b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066613.21/warc/8f9344b32f452ecb963ec96d01ea3b3c456af925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066613.21/warc/c0ca9362438d376213bedb922c386a2ba73c6fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066613.21/warc/e63c0ecfc409482bae8007d19fba686b69662c83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066613.21/warc/f30834d3e017a5da7b8e41763b8f7aff80d21e2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066981.0/warc/bf373e247b2890fec5124f687b8a90c41e3c6439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038066981.0/warc/d2897d8a04884ccd8ceabec2d42b7849fb05e0c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038067400.24/warc/06846d5e9d1c77c23b54af1b67b8d2cf1c73ad37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038067400.24/warc/747a56dd51309b2566f5ccd866687fc138d8f7c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038067400.24/warc/b3414a4af610993b6cadfec05f0ccb5f87d7741a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038067400.24/warc/dd21edd7a7ca4f2b6b60065662257e11528c182d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069133.25/warc/2f85dd311c7388f4c10d9ea0db257e5847833c17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069133.25/warc/58172f0d332107ff295bff56446af4a4263035e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069133.25/warc/827a2f5acf50c5ca4e5846f4cfc67596cc55b442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069133.25/warc/e17047a65d2a76a67a60d845a7cd9760be959613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069267.22/warc/1601dcdcf452253caa1fe87b379fe19c33a23940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069267.22/warc/347428678a0814831f3e250def28dd85fa5621f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069267.22/warc/8c79cd8049af2c21df8b6a43d263667914691a5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069267.22/warc/9116b7a258603d7ac99f24a73d097612ed37ae17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038069267.22/warc/921b292d299117db5d60016d44ed113e10ba2a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038071212.27/warc/7e5be3e025e54eca49c97cb1761b3397480f5eb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038071212.27/warc/96ebb90bdd830f6fbda144d70af67b15e30e81c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038071212.27/warc/b9d1ff657ee8ee41ce3a3fa5222721645d49192e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038071212.27/warc/da1db5a0233bd55d37b785c1077d485b1b51f3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038071212.27/warc/f00008d4e5c64095f2658cfe22e87f7a6cd184a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072082.26/warc/0b50447a3681f388b83f69f27d86a05781af45ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072082.26/warc/105b2565a4071cc98c5c691f99bcb72a0c0d82b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072082.26/warc/2706a31e2fcb7aad5566dd42a01f4460d9e3d98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072082.26/warc/93eaa93c48ff1b3f30fff560730e974557cc18c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072082.26/warc/b7e584f7b525ab776f34636734b3a65d75d8a28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072175.30/warc/4e01ff5dbdec3e171e8a72cc8f83158f82bb83c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072175.30/warc/5396416877d882467d4fbe610b81f78f1e6c16bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072175.30/warc/c1805541045c4b9892d8f816ba1ba57257a776f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072180.33/warc/14de4de4c2aea0c21b49c8e770c63e4dcac705e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072180.33/warc/1d749d84c34e226d56e09d469af558a1653580b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072180.33/warc/7b5432181549190534eb929af56043961c35f13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072180.33/warc/82ade6ae4e0f6c2bf0e7809e94b5329a58e9cdee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072366.31/warc/1a82cd633cfd01884e3f219400342c7f2683a63b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072366.31/warc/31fa1be4636acfd080e47e1869158209df965335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072366.31/warc/54a12ab4562b7e77c88b54c8d26d205dea4de742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072366.31/warc/c1bbf750c644ec29ea9125686af003455138b52f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038072366.31/warc/dd6ba5746040275d70358500106b9d19eb0e5e89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038073437.35/warc/56631af906f83069a5c704569483b09a0b2e9170.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038073437.35/warc/73e09a135c5e37f349c9e04a3e9a26c94c26a5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038073437.35/warc/e502b15c70386479d5f2935d3e44bbb8b4e5534b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038073437.35/warc/e506a73ac13399ccac948d7eb101b84946ce2185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038074941.13/warc/34bffc5811b94e52531afc012b8b375a1e569fd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038074941.13/warc/5983f758bc9655905b9c72b396345e9397f011d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038074941.13/warc/5ee03232a3767487367efb50a68ba92aecfbc139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038074941.13/warc/852cb7571b23063fd051637d599c31fdf8ce1964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038075074.29/warc/0200b1d6044bba261573eb62ceb642ae41aea901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038075074.29/warc/43e56335646c8a5cf3f9873aafd9218fde1ad78f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038075074.29/warc/6257db78e5225a091f85e92dbd4e25181f35cc25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038075074.29/warc/cf5ae2dc33a602f04ec86b334cd03bf57b969f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038075074.29/warc/ef85bcae3a1860a79d155da479a885dff4d14d08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076454.41/warc/4c1fdae8ed5e553e1798795de07f97808736cb54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076454.41/warc/8af2af399a67ec970937c0e7e1776ec9280bbc1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076454.41/warc/e306ab6c1e316589f31d1745484dbdf53d931708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076819.36/warc/6834eb19f5e4919c244306e8fbc8071766b1d104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076819.36/warc/7272508c56a6bdf8edcfc51e19f38a2f4977938c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038076819.36/warc/77f6a32c24005ee21d18cc4916a16576301c863f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077336.28/warc/98b6ec5f0662d9e1df90d4325d0b95419fa2f0bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077336.28/warc/affe7ecc0c42a92ee977633e6a906f97a31056f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077336.28/warc/d45bb4938213faee612de0b832655da2babbe0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077336.28/warc/d4a44e708ba2180479016ff67230661b62488d36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077810.20/warc/031c97a71f8b0a00a0c2f03908eb3a6d3dc5505a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077810.20/warc/4e603b53ac577dba1b31d3a4ff6ec434c5c676cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077810.20/warc/69ac852ec605a2ed85b2ad4ac4f9b1f6530ee7cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077810.20/warc/74ce3e2ae44cf6c727f2441270006e6dfc59892b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077810.20/warc/e1cfc359e858300eeee5a0500df373bf9681ae81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077818.23/warc/3d0eba0cf1fbd20d9ec0f34a6cfa28bc7a21c50d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077818.23/warc/4ff617527d59be509cd0283bcde0fa07626437c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077818.23/warc/52b20d90372f9494c6a5adde7e2b9af05f3ab04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077818.23/warc/67879fa064a684561693c3f2013a68afad0c40fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/17541b3c9195723a586880bd346a7fd42130cd2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/41868fea8af5364cebcf0a9d166935f425a1a57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/6090897b158b9b51baf6e2f96e00e7ac1efb6df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/6d4da11219b13521e80b616f6cf03f63f43857e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/804f101c9920a8ef08ad72af3ec8da510fd1bfa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038077843.17/warc/c448525408d778987801c7788ff97f9ea6e14000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/67b01548099e3c6a17e63c1d1942faeb2c82286d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/7472a1f6a4344e0153fef201b504e058c84afc26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/81a8a3b9b8c9b52f7a65c280d6a3e57a20232d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/b3882491e7356faffd0fd482e591c8b3e5fac414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/c844a55f4ce18a6c4f028cdc5f6992890e4b0e1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078021.18/warc/dd5e61c42b8556983f3200967d58a57e1ffecc3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078900.34/warc/4546c57f682b683624653926b21ddac69b9d6f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078900.34/warc/7fd26bee868a85b94cfb5a11a46d23fecc11d49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038078900.34/warc/d543128f89dd12f1a4d3b26a88c214c1c9a5c3d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038082988.39/warc/0bb7cf567925f59937385f968c4535c3e70ff624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038082988.39/warc/288e7cf70edfe652145f2af29415a18d817a238d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038082988.39/warc/689bcde1d4ef7227abee57ba7fae043847f70b87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038082988.39/warc/c779fa235cd8da232a63e0fcf91d52974f73df74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038083007.51/warc/40a3fac9a1ebee1c63f2185f5e952ca7c5fa934f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038083007.51/warc/577c5bf605c40e422c8c7a811438e0921e4ffc5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038083007.51/warc/8cb5cf9ecbc90a151f949016e387264a0451c042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038083007.51/warc/9d3bb29908672c2ffe8a3e225ffced76a75ea1f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084601.32/warc/0eab38ec93d1e29bc41996b14e48d21a103d52e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084601.32/warc/325c3fef8fa021f3d956ec7bd5fcb849c426aa78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084601.32/warc/9ccb819a5e7411cc3f1ba4b875253841b88d7457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084601.32/warc/cae3bb2d8305ec2e5ed2aa1b2c9877c085d17da2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084765.46/warc/0ba75e4b21bd0ea1c8e5c654cdef7fc74cf79bcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084765.46/warc/5a840cc08c2802f4dfc19acd36f272b135f044eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084765.46/warc/b5784d269e7a9bbe5cf8cf7982d4bdfcc2741b10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084765.46/warc/dd1d992b51c5a7c1385b70f71325d11c16e35198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038084765.46/warc/e3cd9b93e665211543fc7f9a26e7ed381d8a093b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038085599.55/warc/049ded267e95afddf1f91daf02e3da717a6204b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038085599.55/warc/21c56060f0844f74f31bd63ecc6c5f2e482e0540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038085599.55/warc/a98aa70d2059ea9bac408f3b6a7746df88abc24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038085599.55/warc/ec63c69d30dc761f3ca4cd40810884ae069926fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038085599.55/warc/fb1942ac9f29997a403b50fdd98685e2fc3771d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/220d1db90abdd9cf538eb8f2f80b619a52efb88c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/3c9c2315110ce1e073639e995f96fd6cf8584aad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/6867c73e3debf4583475ee1f64db3fc42a35c7a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/c3e35939b667053e76b749ed633a0c96c3cc4a68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/cb964905bf52db3eda4413fc59ac9b636c1f1f7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038087714.38/warc/ede265485ac5d2039276d6a556279e6a1e409ef2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088245.37/warc/22864d66611a29f10a7659447b62e769810fa88a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088245.37/warc/5887c9586c8b7ec99ce2997647c1c475cc37c4fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088245.37/warc/75a3c87abb542e653fcaf86276af689759b043ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088245.37/warc/9d1690c77b9ad95aa72bfcfe268a40b056f479ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088245.37/warc/fa75b7d734794fbaf05388deb674c88b593f327f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/315e511a81a248f49836dfb003df6b964d45ce0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/49e54b988bb97d5ea0c641d7cd158564bd1cd301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/5514d13bb8174d68fe10cd964b0f6e488c20bad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/621b885cce31c8c22f68f78c79e925eea599b9f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/bc5049618ec4d04c00eee4217d1382ec1ce2396d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088264.43/warc/c06fd460eb5362eb07c205e0c23dec2b5986d019.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088471.40/warc/4902763365d1831dba702121ffa7729588f93d9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088471.40/warc/49fb7ce601e29fbe0ffb69659f50044a58fcc65c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088471.40/warc/7794801770889c2db5e57fa01ffcf93793a57f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088471.40/warc/c964c4ded3f2d06b492d4c6a17de2a362f7d6283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088471.40/warc/cb06767509d58c9f88f6f68b2cc89d975ae594bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088731.42/warc/712c47c7cecbdcd466cdc79049719b9892a6b619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088731.42/warc/75166437fa89badfa8f79affc263603915cf3670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088731.42/warc/9df0ecf326419b5686186040905876e4eef2f501.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1618038088731.42/warc/d268795e33b74d3cc8f6acdb8590b863ab737818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/169ac96eabac86f97587d97bc2921f12a8c3c272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/1ea3eed47092334d78c4a9d73b69d34fc9855cea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/205c4b3d55bca0f528b2d37e2703acc08d786356.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/26aa35e9fd9e8c55e08b10486a82fb041c12e738.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/2a78b6564a479114be6bd6210ca5e460c0f7557d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/2d67f063f668ed73f1788f97c3125d2700327fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/334986b40abd2b89c8aa94da34cba6eeede90ad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/4845985e0d64bb06fa4b35932586dec5ef9a097f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/6123a796da321dc97af14ab4e1b4f6b3fdf6697a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/679d43473b463d80c97fbece98f4c90feb0e2ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/69a606f7cad41076f83b076dea76fa774f33ca5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/77accecbe890f780d054af8dadfdb4cb522d31b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/7930d029bd9d2051b7d682c6a44104c8e47dc529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/7b4348ead441bd4a04fe19265ce07c7001bd7826.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/942dd5421c7adf4b24f73ed29cf1653a5100eadf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/981cd7e1d64ea220578b97123df8d6e4fd799d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/b0892dc2a852d66a3ba974f7ed5e1d7a058b729f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/b542f1932046e081383d015e8f35b4e8c89efb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/b57a299af4bf02e57c11279a640d81ee4ba93ea5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/c38fc5421684e5117aeedec1e85fe0a92d99921f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/c807e63ac006507bb8f81d3245f4d7b825c5a5e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/ccdcd1031a5e70b16b096706e7ca4246a4ff313a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/cf5f75f78306355f8b33727c6b6b826cc5601bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/d3a2a155b21663b7b093dd6be37a451da2de3de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/d45380191a19770285139c8a06ab912fc7363e1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/e1ca1d9c43ee0e9ca69ee2a9ea5b6b05d110a189.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/e343ef94e741c230f921da0e25bc1b29de24a072.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/f666541b6878f882fae0c276d5e836ab9929ff28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/fc16fad391c56ac4a4023246a285b7def1504418.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-17 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2021-17/mix/input" + output: "/tmp/cccc-CC-MAIN-2021-17/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2021-39.yaml b/configs/cccc/mixer/CC-MAIN-2021-39.yaml new file mode 100644 index 00000000..ea611b2e --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2021-39.yaml @@ -0,0 +1,794 @@ + +streams: + - name: cccc-CC-MAIN-2021-39 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/0065ccb72a390dbf3af24b25bcaf1342b8e91539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/012aa3658aa001de083a9c9f0cd8ac5d36197e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/07d2a5e2e46b0d1c919f4a9ee3ae1144aacf0f8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/11b86319368e0a1759eafe2defc46622637acbe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/11d823aaf342e048db34b56f6f345da37306e3b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/15c91eab2b9e79d48732d883aa5df92577cfc74c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053493.41/warc/5bb7d95087d106c093a0884e43228310b11ae227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053493.41/warc/a76d3c6e2bc574e939c313529269793a87766a82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053493.41/warc/d7143d94130e2bfe46c230a7e445cef85dbd3b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053493.41/warc/eaee95be3f589e116c74666e2d919c99faec7c73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053493.41/warc/fe21e74128603be87bbd29e6e0f4a9280bb9aa09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053657.29/warc/143e8b3a414606ad0d0f544f3865fbe544ea9a35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053657.29/warc/66cbc06eb20292650d1cb8c8b293503cf2d183e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053657.29/warc/6876c0c439917d476e6fe64a1058d2c3e890156f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053657.29/warc/af120117aa0fa756b8a61ca2b1343f2c96d41355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053657.29/warc/fc179c96c6ae6f739fafb0c7e3f483939f33ac3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053717.37/warc/2ea6e46af69514c106d2847c9b94daeb4f11472e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053717.37/warc/6b0202c0bbc3077adf3b21a14a5fa283c1297088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053717.37/warc/6c29212dd73ec621a4ffeae4d4931556034a3b67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053717.37/warc/8e5abe6fb55a796468706f5201e61ba5da85e529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053759.24/warc/33b83f8277ee5e6003079f896599608224c733ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053759.24/warc/50c53405dca569db69847ed2180722d9878b7d2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053759.24/warc/c0c7d516cf656480d7c694950410522c13bde9fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053759.24/warc/f606bfc0747d51d5ee9a512959f4c6b901d22139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/22753cd342998568bb68a4c064a30f3cf574e2e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/5aad12c43c6eaf2945a703cb9fcf6e41a2463f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/6ab195ff82593511fa93d673c2c0874f0568d945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/7bae6e8df50274e4c963fb9c22a552e5718ae4e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/9088d6703928cc681aab57c41684d2e1bc509b6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/be9e03c96dc767e4309f3d07ebbc1d3f5b4fbec8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780053918.46/warc/f68dd6aff4cc629f2590b614da3bb1117a3c7838.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780054023.35/warc/785a44ae4893aa2c28515c8084e615e73dfbb6ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780054023.35/warc/903bb1200188a24eb5fdbabad032a783262f098e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780054023.35/warc/a73b396c56b7e9fc3f9ea860741e8e79c6dfef89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780054023.35/warc/bf9d629fe4f6a8f71f6de5f9af7851d0ff2ab071.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780054023.35/warc/e3261892089b4925c84c9911abc779eb7f4f1c16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055601.25/warc/088ca9093e43fa5066576fd1c8527eda7480ee3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055601.25/warc/716e922ef1029aac83fbc5626ae9eaf9eac64113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055601.25/warc/b73d04a32ede695ded0703be7269d9a5c263a131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055601.25/warc/bdbfb3c93bf80a523b418475ae4611e95b261ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055601.25/warc/f91ef1d47f9fa4735d25f4342f0cfc50d667a3bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055632.65/warc/048381c9518a08ef9968e427ca7d96c5cb30edb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055632.65/warc/1a0f36f4e16b33d73ef7542d2fd131ca34a53769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055632.65/warc/46b141e30b2942121a1a62737bd8f0ecff06efd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055632.65/warc/f67548e79db98f06b3bd43f7e2cca12d43d18a20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055645.75/warc/884b6645d964b2452a4058d83d212f34ded79176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055645.75/warc/c23094bcc2a2ada3308959709593c22fb343e8b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055645.75/warc/e2148634b49946d671b07dd7152109845e680f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055645.75/warc/f539c239be3221be47d651f10d485372da835559.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055645.75/warc/f65b3d996843607c155ccd4df1048543cf4b4e6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/1c8d0f040b3c1d3117de3462ecd321bf89c63544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/623494cc30242aab97241ae4ef6996500f65257a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/6a8c6f91706f19b84ce22f4eae9160f3b9ddf9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/86c037f404f781a914a866892c31d4f866aff551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/acd41d5f175bf8f089e5e130e1b650f2b21a0e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055684.76/warc/c8c794fc37eea85e3c8a87aaa64baf3b242da388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/1f20a515b9b23cbc2ef6671caacc08583fb5afcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/31b66cb94c5c785ba0a4aa47693f3755b03a37cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/3aa8e43f7aff2446303fb41bf7102d14934b59bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/62b35cb4f7a5d79f93de8c8bc43b8cf3b544df5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/6a3fc1f37f91dde36e38009a0ed644057703310e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055775.1/warc/c6862f1db3259317114f92836e7f400b7df3f0b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/1091783f8356c9a4b7bce7b29ecf5726312e9787.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/38cf10b84f528cf2357337000e267e1edb7d3a6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/5fca4c34e4307184a3bff8c62b6a28e930cbbf8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/a80e7c21744fa7eed6db15f0b7e951ed48d5238a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/f0d2808f0e1d7a6981e4c58319acfc0f146fd2f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780055808.78/warc/fb33ee47ebb5b37fd00a226228232d374021bce2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056120.36/warc/4c83a12417c7322130d4a0019032ad018990e4b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056120.36/warc/58dd78f7d6af0301189cafebe0886cfe2f5e5a7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056120.36/warc/854228a6ab7808c63a1dbae424210bd859e92785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056120.36/warc/a6b426181f74f037847bceef9c398962e8883ca6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056120.36/warc/d3cc7a194a45b8c101132ce5cfe5fc908a549ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/34a465d2e37160fac8c5860bb1fce091f14f57fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/46f37939fab7c82cf6f0f3c63d94527a28c95d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/5fa5343552587dbbd7e8b75457fd3617d52eba59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/6825c81a5f8370cdae10e3066cee4f7e0a0c3074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/895df609835b4d17f2a8225321b7a5d54c7b3c92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056297.61/warc/ed195e6e43864e0edfe1eb9d7c71b66051f6042f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056348.59/warc/33f3c22507a5cc80861b3a2bd25fd4ec993a32a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056348.59/warc/676a7837879d7db3f25c1057daf1e4a67e8902ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056348.59/warc/a26094accb38f26a7b37a120359faffcabd74e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056348.59/warc/e501b207d1e9965680b00a1c708abcbf490e3170.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/033c94ff592fb70c73a09ce7022b00255e0fc292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/42461d8be9c49ceff1aae710107881ab5e870f2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/9d8fbb8dcaf0e77fb43565f6eed6c6155a622dc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/a87a188c1e95d7185c71be2377a9a48c055385c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/e4ec19c2d9a0d66471bec48aa11e347e4e9835aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056392.79/warc/f18ff93d94bea6075d06207be59210a998925cca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056476.66/warc/e96fe5eb66541411b4489a9bce4479b1550f115e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056476.66/warc/f43ca2b5c4c3b28f11d556cc48a267a1c9bacc98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056476.66/warc/ff1d04423c4bb9339610d45caa4b0aa2bf7d07c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/19a5797c42cab5dba2d38607bc4cddc613b709df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/7cf71deed280d05e4cf8367286c18032a5addd94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/816fc58e81f60e883c1651cb902ee33fd601426d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/ae397e50827795a4463bbfb1d794748887765ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/e50f7a48168e064f1ce9ccfdf32c4651c3ab131f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056548.77/warc/fbff349f06e6f80e7006f15e47f32014aa9a51f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056572.96/warc/1aaa618b0088b1860a2d6991f40be936d89b6740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056572.96/warc/5ea72087d2f45fd28ebca2ad8ecd1f78022ff1d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056572.96/warc/ca4b2bb7885155f72ad4af09ec5acb3b21e21344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056572.96/warc/d8a234a64c97523b19b6c87603b2a33b513c418e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056572.96/warc/e9cf9b36c67b204ed077b14e6c64fc01c400158c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/137ee321676df71e5fc21092b264e0ef9eb66d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/560c60cbf1163d7767e8f2047a500eb069e3db02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/a640782d8e22a45e81b52a66ae5d4df4f311eae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/a7aab91d5901443746637e9a56d07c0a4f2ebeb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/ae285d73222d0495e37376a31d2d6555649aaaed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/b2c1da30e0d52e3d7a37593ba4cad98a75756c69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056578.5/warc/c43f5505b779de24e9b6edbac84c43c3139154ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/277f59dd5bb52bee949e94415f5e043274ef2985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/32faffb719817695bfaf0394e94c2fc49a07274b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/37d6b6a0832d24ee382d59eb72a05eb6aff35d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/b7f38eead74fe0723f562b9b4192fd591c3392e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/f23624e999601c9b8d6c324caba8020130296439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056656.6/warc/f55d0e20bdddbb29c326b4be05459a317e2b8a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/01507d98322ac0808333f9bb31f1408164d89e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/2cde5cf072696e67ca0eb86c38d888a89996da73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/39562b4856a6e28bf04e92d699c109c5d29d4207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/5b39d13f278699f165c53491e605d857a26199b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/8c3b64491340278f539de6c786bd0c0d283cb10b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056711.62/warc/ded0a6083b1757e4a6629a8bd8c6e9d02fbc29b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/02fa21a0a24916cf563344d763bd5ba3e66cfc7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/1ff7b50699cd777abf9cd92a628b04e9cc66d06a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/4f1090a8651ab68555e3cba5407dec1a745185bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/af0cb98763a27f61f9812a1ef9183988d39a479e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/d344c8583867d361dd9d110c7677748f504b75b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056752.16/warc/f092671e5278422be4bad62b75938b5469222ab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/0c5d5d67f07f0fef78245607992453fa2765f478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/2a20bc1d8c04021ad766e2aa24dabdeeff604dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/3a776ed34584c7c5b8f525a3826a7225ea397898.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/6648d431741d9a88855ff37d6d7f2631a5e73cbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/6db9688145a85b227f2129a584cd7845866a99ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056856.4/warc/d4cfd6f52bc6ea498378aa9a0b90d56ba08d5085.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/0b36c8dd7e9a6c577c66c8c11ad5a3cf3b4b63ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/160dbf90501cd8dfc347fc058257d2b8c3c437c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/2d5393d2b256e676287aef976ee62b2f9c8da359.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/3a15cdf249c029df188595d66fdee56ff43e6620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/644c26b422ffc71cb59f0cff4c774f3dc62b737a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056890.28/warc/758e2603f4abe0d17903b6e84107de7079f9d5dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056892.13/warc/645046139b9db7ee86c7059ae8cd254686c3dacd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056892.13/warc/6ed2e20c7ed9c12fb42353f1e0ba71bde26d9270.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056892.13/warc/cca8afe3e8409f651dd964dd5e425edd2aa5a751.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056900.32/warc/0ce95e4ea3b4d5da6f37cf200b2bb46bdd0a86ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056900.32/warc/0e04c3b746080dfb579f3550440363b69c9074e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056900.32/warc/193ac12f21c15adc7faa7737664eadb745662711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056900.32/warc/66cf1a1677388cb2ef7ee23f3838dfa5ee55ac00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056900.32/warc/faaace150bcd81d79b088a0dde93a81ffcecfff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/2ead3f59653bc1a3bd3ec8715f73f362d95fc985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/469be77177ffc614b4f728d74e5ff1f315c68038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/931fd456341e5fe89cfe3f5d5a6a2681e37b0cd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/b3a2ca24350504750770bfc7cb62897b308b7e87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/d73271c60ef2625fea8e128497b9ca59e8ea8dc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056902.22/warc/e7e2f60389d9226ab850e3615fb5998287b2cb32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/0ceb6ec4821a7ca00e55c65f772e885dfdf2e47a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/71885a5aab9e9db9b57e803fc7554ca2037fb0bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/76f6c11077362bc47b90c015b0ffc49f47df41f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/978c5e77d49a248a185c65a04508111e5d2558b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/ac1345297641d296a0dc0192d53054cee68f630e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780056974.30/warc/b864598a1d47f3ac9be225558c8ee833e27ea3c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/025c90f0c4c814d45453c547e0b1ff670e09b3b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/1d57b7150643061be552ab7198cb6e59fe6c3f45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/2d748c87fe88da1fd09e2b0ecefb0b5b0b4f0fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/659f1032bd6b019c7f7c8643c517792a637cef66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/b24e865d2e4b26d5077b3db654742f4247362dda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/c9b91ab06b714ad6ffbabe7a68f399034e284d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057018.8/warc/d0b8cef1483a32fd08ed5c6b278d2551403c63e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/42227e996caea6211afaa1e9beb3cb68518ed1ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/53547ec774b3d48571240d4651c93282705cfd4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/57580a6c8abebb621b6010bbd29f59b8e36cd00e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/8e1755cd966830c2387b291b357227c434912914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/a3a25fd0b2d426be031ef3c47c11e69326d896b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/d76b00f5128c618a03a1bcce40da33f396be1a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057033.33/warc/d77dd6f2b994c08613b82d69c6d4e5d1a6c3e81d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/223e89e267fec0c5f13bfaba0f89f3a779d5cabc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/24914bf74850703cee59b7805d53489196934d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/499e6c40856c8e6497de94c8dda79ea8a9cc5f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/b5389ec49d299ae3d885e3297a6d95486f0cedc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/b8add31ea09d114648745a6c404ed67acc661752.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057036.89/warc/eab7fd6c50bf605d9c6dff91f0cba2871ee6b3b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/1b370d22c08007eb8fe0c953195ae8b7b488a662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/3ea2e1a5fd311edb1fa0cf2c1f02bd6691566a5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/4dc6331c942bed8c971253ade4acbe7f6b64a547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/60c12efa4fb57960878fec51b995e7499ef41209.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/b7716eb79be7a897c151abd1e7c71d5990e573f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057039.7/warc/d6e824bc844486deda253e48add4f3afd9307439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/030bc2ae06051960fa21ac3082ddc3687356fa29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/060544ebb51b089e0d78e7914b0212ab1c9bfa61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/66ae21c0445c0654408f604dd0f6d43427012f30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/9ccdfe0e74681b147f514fc7519f58d79e23ff05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/a62ae462073a636bdcbe6f53ed01ff6947bfb1a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057083.64/warc/b7277df76fa75e977fbd0d8398a816e4b49173c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/0cedb9f24582aa336a121dc233c878760ab0373c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/21620ab863d5d825cef62e6cf5968bdb112d8763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/2d8bb90f2b13c8c60e119dc334e49231a1999609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/37ef722f8ebc00fdbcadd208b0fdf94fa96309d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/5a67718915936d372aa65d7378f410e463e39272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/ad583aad57e65300cd094269a76698d2bfa38280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057091.31/warc/e8fbc8e5626fafb363de60053d8c6239706a32d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/0d09386c4d66fb11a6430fffb32491ceb287480c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/34a58d03b84f617be0ba36bd96244c784ebf3712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/3a8c6a332d171efe4c16a5cbfe42c1279eb62569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/6029fb7ac8af4abdda60a0cd068b39e1c2a86c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/b123078895e4184cd07d4465220ec85c86738fd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/de7ee39eee9af96277dd754bdb1887cab0ceae58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057119.85/warc/df0aae2a95de147f3b4b05b40410c294e1622d1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/1566b6926882e3f81a2f67bd32d831fa196c9904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/1a8b5a5e99cb705e4d5ea4bb578aa4d16ade8875.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/3855c18fc3770406ca161b0d6727a0ebd5622920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/72938884eb75d16b2b88b1458166fea2551a27c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/b633cb8bd920b9f1eb1fb4e3bb61ec4f34be3d3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057131.88/warc/e09d5bfb3c01fe95f166a045190830992c96fc4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/4831ee6364acab1867090c2da5c75469e0ccbcff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/4feeb5f2e992da2a3da53e8a06e64d0cf5896c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/762482761147c403d623245f07fb1445e063a69a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/8f004df40a81f2e0e4f83636cdb81e7d56517574.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/c390c524122be0528f30922afb0e9b815aa87fa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057158.19/warc/d72928a21b14a57d120c4d2867fea3f847ee034d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/06a81a0c5379c27ee807f6e1e9c51d29ac4cb631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/9efdcd00adc833728a240fd3d93cdf6d65f5f3ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/e54d939810c07ca7dc968e931258470ff07935d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/e9bb0442b458a9a2f120a2241ffb9160df3e3f29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/ef7f3760d079eb9e9c36372cd93c5b736020da67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057199.49/warc/fb3424f137d292b6c84d13fbfefa9c2906342cdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/155f10af77ccb9696215ae335695b697af98a6cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/2f077e7d0c873c30fd2be28d869b7619de0c6e8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/5e45331c467ccdf9310d9d8f545e8bef26eeaaf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/7d1eefd3522c458c6b8aa3e1fbdee99ceb71bdbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/8932b440802acae163f403b2de7e9ea67bf1e953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/c68fe0af6d20ed575f1d6d540b348339ede70375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057202.68/warc/ccc83d1e8936dd806e942481000c0e87157c560c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/0ae6efef97c82f8271c5487b7de95851b46b8b18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/3b7a02cc7e82ba07f8b7228ec20fddd677ced9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/46b3a183a5ada7fa6c7110220cef7a28751cf264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/719efffb5bee916a34d4df5817f0f5e753c98c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/94fcfe4f708f5fc23b50af06bce28679b4a4fd21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/f65fc8618f3d04c60979edaabcd887487fabc313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.38/warc/fd76b95f686e9ce20e7d29ab48ba6bd29e5debe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/11ab8dc826d388b05bacf33ef6e6be6e30f3537b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/1b72b561ff237ba1b7e2b4ab3096ca91e6f6b11c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/53efbabbbd706d8ccb9607d5f6954c522cbf41bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/68967577f4d9aa63369e293f954d0f6c272f1b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/7fb0ce42ca666e6b9a832712ac8503ab476c5f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057225.57/warc/b96a760691960a1ef160026d06d515c0cba1abe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/0e7c351dbc9fb108a56e804b1011cd24d6ed11d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/1b7a29bac20c3e8fc81fb3f323e8847c8195afe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/c83e163e2717b22fc95c7b619539157f9cbad815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/cec0614ec62fdf9cbeeac73c8c09ddea00777164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/cff758c481933efb61b69c257a354e045eee4aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057227.73/warc/de298a82471587c47f13dadcd8c7a6ec4c064763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/02619bdd266bbe846ccec54aaa63c37e83b13ca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/3cc097ef7a1c77a9737c56ab07a697d38706643c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/485ac0c59e8fedffd548b0063389985d6038bb0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/a401065f12a5aff8e13aa1dafe907226ab1a35b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/b2cf58472b837b58f3c90ddaf39d2f0d9c09868d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057274.97/warc/ddf9019780beebbb6ee00391e2577c98a593e821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/35bd774d75f4095cb0eab464729a14c0e21ca56f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/57d87fdd6c2e59de17ddaa1559037ccd5a1dd30f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/8b311f07c4a92a2948d0a17d8f4a0a86f3296492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/9c4e166918b6bab62a8186320cd02501f385f6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/e0c7c349d05a119643931e9881ff9b00277506ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/e68e99ebce39087b516b9326a7b49e25598de124.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057303.94/warc/fbe9ff5679c036f4b6b49fb5c184ddba120a7862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/1c8c35ca706ac1c866cbe0a0494861ab074a1f04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/28c78fb8e1ab72d36baa0957f97958f97e1454c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/495115edd523fd1480a2a184a78f5442639b1d54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/647d7b9f659061ca10a1175df76600944cb5080f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/bf5736625babb2d6ff571c90e4053708ab0294e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/ccd9b496b813973560d641641eef4f955eae364e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057329.74/warc/eb5f9e40ec0159db2c9e54c8bb2c641a0889f855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/4812bd089bc769d16173842c33a5be0269f484cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/7f3738361a68d783bde2fd0401bf509ef2736a61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/b20dce194e5167f56710377278e3b7875cd71568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/c133ae50f9d06c176929f8870c9ccb71578eb121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/df7b309d95eacdce485edb9a7a00f66541506f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057337.81/warc/f14af009f9d1b20ae543e4547445eda12488a69e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/3f3694dc8ccdd91ccf2ab80ef82c6b390d832742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/7c926da75878b4a59afd6cc8fc26dcc08c9871cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/c2180e491af23bc12fb590933c30986df1c72614.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/cd6eca58a0657426c5138bb6f2b2ef9ae19771dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/e55c53a5bc8c510a5601ef2a40244c1ebd7a1948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057347.80/warc/f7e20552ec817e7d9a2ffc2d0d159d3b2c83f1fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/1d6b67e3038222076139f548599173d3c165261e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/56fc9ac383972edf67fd72455bfd3a22800834b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/98232745c922583464d241dbce524ce044537004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/ccdfdb83529587c54e580ae27f7d642ab9c9bba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/d1adc382f37200168fb5ecf407b006d28397d6bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057366.40/warc/e71c44110b4ec3b83fd7c9490e7f93883a1f13c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/171c3d7306f44e749ba18fc3458a06089f47514c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/1bcd9f8ec727d6faca57b1513df5245c5e16b8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/1ce6a0f93e9251f36519b88c73fe7651c53d0ec6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/2a92fb56b3ec3d1b1c6febabb7e2fd5cf0d20157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/d80a508d8405a89c31a4868a91aa31143c6c5e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/d865c3b5bd85c4929917a24d9cafe74115643af6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057371.69/warc/ff0cb907affb9e97af7fdd03e2b231ae7a87c4ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/11d61c1111e0511877962b946c719fbce0bb1529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/201fc97d0a303860c205034e7fb2462c00f2ebd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/2aade69fb436e082c208704f86f21b953813b0ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/38fb1ca4df16bacd7887db09d1b1c226048fea42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/8c899d356a5563cf7ff093f6be5c20509ac99c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/9228d8183d73d416fc850e0f37753981f5e1bdb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057388.12/warc/cfaf2d979368ac961ccf90642effb90d71843d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/0b92fc01eb7e828732c2a62d46e14da7d965c492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/51eb6f477265999c36bfa31e7b18ee13729192a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/8a3757494f0a1073943a37e4260754482bb74188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/91b2363695f5b9b93b5153a11040fcb338238df0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/b33e07ea8999a32effb9d0f028915c418bfe3382.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057403.84/warc/cd4f43abfb5bb5bc6a396da06f43b885ec43230b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/25e6d64f91b56d01749c5775663bec0501cdc5d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/2a15a457aefecd3cdfb167eff1f97a73e4df86d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/2a2eb7a386eaa3451bf26b2cd4b7137a230790bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/c352a5acb18702ffcdd12f689ae38ef959007f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/d283d533b3cec59247d046ac2368a0b9cbca956f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057416.67/warc/e1cf497839e0005da39a72f20a0f3c12414c887a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/37c11f1ab25a32e7f3c11191aab6e798680fd7c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/97eb3882171e8e936823f93bdb22c3015eac0059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/bbb85e9b48b0fd86af22326f148db20372c414f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/bc4807beae1b2d8c7b9e7eb7589881cbff5c9705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/eab4d7ed029e6dc66a9df4871bb19a17c9cbd7ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.10/warc/f8d8e22a4ec4a4b9350e7ab4ecdd5bced163d9a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/33c42c835dd1e62831280919b01ff078c496cb93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/44bdc1d8773ca409ac38193895c53b787690f84f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/6a33c0211c2b0ad387ff290f1f105c8d869bd73f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/cb4a7231e0ad8271a41a56f1d9270c1a7f8d0c95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/d6574f9cc7460295c2c53367c237d281e042c17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/da4b97633d11085ae6f61c4543bcfd58aaf70840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057417.92/warc/e263b3f096d183b8de692928c625e8042d680978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/0278ef4ed1cbf8d31fd77cac266a450a90d8f323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/457d3504817b007451ebf79d68fbbf69a7d786ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/6b8f7e43f51255e04e902b3a5c3bcf66b811bf59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/9059eabaa8475e3defa3335d07f81e01516b0747.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/92f7d059aa20c204c833b70d20efaa85efd12f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/ee9cab403471b03e3d66c6460a84750c5f96e212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057421.82/warc/f283ce2de7a82d05f7c9b9db1488aab43a142a0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/15cd982fd330a86c4b15ed15658bc36fdb8af06c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/9b2ec072a1c375369005e3ecb55d3cae799efeb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/d2a3baf9208cdd8536e33c3cf742249fa538ff83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/d67f9048b5d51220df4fc42bb0f7f6dfc9632cd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/fc362e7a2debb0f06974f448a223b716fe9a1ebd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057424.99/warc/fd801402ec3024481753d1ef29d1d49747c6497d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/301e141c504cd3cdd8592e7295399810a3ecf976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/6565c471fa977c32f1ae0e7ac6cee4d3ee7a2f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/6f8a5e18af9b4404d57b9d8cb9fd760656710833.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/7fc4abdec1262d5c35598f95697719111855b9e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/d201e1a7525ce6b00cebb21693a1cb8304073a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057427.71/warc/d991bfba50eab205cbda585fe0996e38508bc190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/0083b84199e5bcfe71aa9e28c2ed24192958b28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/26e985953c2e258dfe3decd7ff7f6a28d9979df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/6e47a4e259d675c5a11c9c4d2d894dd3a96ce725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/75a12b2f86a18bbe852bd9809e53695a90d82636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/99a82461fde4e26f2af26caa6c44c88ee30e57d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057447.52/warc/bc70cfc6b543857227725c793f5b4b1ef9bda8fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/02f328966789620d0b52342f39ed7bea6a108ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/22b1aa0e16ac28265926e756831747fcf8ea7c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/32b246f5823880f0398b892755c6fe720528ded7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/4786080d6cd8ab16bf62e43627da559b2a909d05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/69b25054c43df98743109040488fe0c7332d39c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/7d8cff8a9ecc76230f113e02915e052aac34878b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057479.26/warc/f23617df70cf971120ffd64c6cd8a292cd9aee05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/001218e599ae192f4767dd13b52d6f45ab0e32e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/308f5b12f686152cb35901c89398f2b0e77a548c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/347033a7cc94806d2efb5502b0b702b3780b8e6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/553c726e67823358d8d63caa46156b1132299d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/725cf98f08fd701959449259b7831d59b25ae485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/8348b0088636ece83fe21fc4f5dc73d3a26d0889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057496.18/warc/e5c7ec93acf81563b587a88863ca84de3d0899b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/2380b13c80e23591e565efc6c23280385035b165.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/39c88000daa2917bcbe2dea9f971ceef0a744176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/53acdc52f0ca73639ec9d0715ad814a086cbdf96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/67fad2676cd3e3ff5d8277e8c8fef0f39345e1e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/6c56eb583a9deaeaf683347e44b5a696e78a9973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057504.60/warc/c407d80c279c6e32ab284e8eaaeefcb92379627f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/064a657f30215ce724f0ada50f1e03ba8e558a56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/33526b32ad8cb7d8671ae0f5a0f1e5f6d603ec01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/3f5681f512e77308ab6001d2893077e5135c1eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/81895f05d4a1a04a81a2ca053d0e586c6550aa70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/ac72e1d586d142ee73144203aa86c297f13c669d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057508.83/warc/b1a1a73eaa32387b220bc2a32cdbf9eace801f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/15ac1f1ed96a3ba1e8afaec4a0b4b2dd60b773cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/7b90330836b28fce59807bb879e7cd18615799d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/9274b1af5bfbb77f2bb195cf1e9060e830709b03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/962582b765277fc505df875c422332a4c20aa875.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/a2f56ac57343009c26c2b5546cebc527bd1d4567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057524.58/warc/a6010fb28b675f8b5e2cc94cf5d85127c9b91324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/0ad62451066833bce548bc008cd403e01cd508f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/22df456c89efcec643de5d6fdf6ff7cddd983148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/5bea1764197220a0de6ecbc939adb6be05224ffc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/7e6dfbf357daeab5eae4dac14307ec559184452d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/9b2ee9903753f7c2bc3bc914afc42bb74f4fbbe3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/a2b72a99cb49166dc07838160887eb2b0d1f505b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057558.23/warc/ee34325523eff7ae90a143bcb2913205e5c42612.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/02feed2f9cf709ff5c895b929a26627ae32f4f44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/0c993a81a66e29d0d6a5b007baefb7ab9c4a405c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/2b16acc219cc28f48b350e0ab1857737aa9863ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/5b8d905792b2728700b9f32a96b892ff9d80c7ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/9744beb76947af76a8621cb11734703dcd0c4415.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/a0de38676905e504af1d4d675122dce2fe6b3297.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057564.48/warc/cf6359ef6afccc76d432afb0fd4336eb959bd5d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/03a11762521f6356025ce3e055ab2c4a5d2e4e58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/0989ec41bd8958e02e639c04c762502331ca405e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/3b55df9f71f2a6f929b778df507448cf1dc9bb16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/7f57def2f77e3d51e42b3a085addea53d27fa73c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/fa76ffe57f7b15ab52e708535c08d8d6b7505cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057580.39/warc/fd3768c89d7a052444542f49b82e0158f9466c5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/2fe9caf2b9a5ae2c04256bf7ad57df3a0bbfd62e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/5abf88cb3baa975cb6c31564b2a5d5797ac7d3f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/73cfd74be9591875bc6cad55c1d523d7eab97901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/91ac07e879d1f7201b6c5305e86d3c2bd3349ebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/c6c9901ebdf9618106b6ad8e532dad2991878815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057584.91/warc/f9c53be09e69b11de71e3ae2f965a3aede777abe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/0975eba0095409539d87776167b0704d4a96028d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/81e1b2c30679278720650bc2f98530d536a654d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/838c58a145bdbdea548a42b3a1ce23c0d6fe0d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/c749bff53e1c59a96bb5f3abab6564574f0f0595.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/cb433f753c799604fbc10a1075937778d33e9d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057589.14/warc/e42a3aad035d8fa50cfe08adc5dc188c20672098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/36bbd4b31880b13f009392d5c0de58c7c277e8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/47078f9a2e04e1a245ad889b1d41860058a7ef58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/8529f2e1ad3a1e8ac9a01ab0f017433ad196cf6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/9366f1a8f78e3219c3dd471fb6461f5d8eb95b07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/bfc1948176cad48f033274e9ab137e8423f8bff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/cd47fdf3c415571c5fd8cee4281d1bb8a7eb9cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057598.98/warc/e53ccdfe8b680aa5d24c5764243369b2cbb7a4da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/0bf8dc93587e847ab708c268cb813908a7f84346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/431b4e6582781a7d3dd8549fd48e672ddcf4080b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/48e46fdac59b9306069297e4b357f1332d998460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/6f303061b616a8ecee5bf99541a803794fbc2591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/7d4ef5bc2aac81525f0178bb790e21a94dcb8d00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/8a8fb3dd7ffca6b1ef70781c3eaf142706c91b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057615.3/warc/a21d87f03eaecd1828807c898379cd5c62125547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/2d3407766a28040a97235ab5c358034d6ab842e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/3138b251f9aa89b9da301d237a0de39fa645f188.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/51edd95408abc192a769f6084101033b63328087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/a2e20c36e39cc7eb72cb3273c320d5c0536e24d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/a58a18fb3b99b7e7bef2fe6f6d18e7b748d0a7da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057622.15/warc/d7492e9e330387741180d718d681be7cac5bcead.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/273eac31960fdd96ae2fc56cde81f7bef125c38d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/2b449777982fe6098a2e19c15861ef176c0cf034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/3984857fca0290799933bd3b0cc815b5d320a4ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/3b773f5badac4d42b123a3dd1e3add9c6ddb1de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/69980747bee40145b5391d4c04f456ab9f24d257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057687.51/warc/92635e915fdc798585b82cd9d4dc098440448a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/0260d549c45146495424d868f20209b36c3b301f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/2b2acc7671826c6371b3899dff17c139201dc39a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/59effcc7c4da7b8a9530ebcb9663a5e1fb0e8d45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/6531f0a7fcc92910cf47799b0753413d1e8e96ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/87349d383569d286e3dde19505779de1b496a2c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057733.53/warc/b6de8feb6178da53703e50982ff8f49320f19f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/0550f4ec04b9fec2efbc34096cf3d6961ceda300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/07eb01d0addbb1d05b78881025ecf68897073c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/18abf922922f2bf11aeffc3bd2a929d363682220.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/19ea7a4eb51ccf8bab08a18f53351e3326bf8154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/a0d29c6947c45d6f5f4e5bee7ad843ff6b0219d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/be0fb5dd19fac6b43622280dbff78c0eed318287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057775.50/warc/f4e6dcd32552181f7f5d45200623f8fff6b06fad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/0b1bf3ca2ce1112b0e153ab3694f46da163a75f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/1a555cec534d212c20cde5b5bf4ed5febadbca5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/1c37535e76973f97834522742c0f2cf88603999f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/4139a81385fe78c6fea5266fd2cd4d33e6349392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/83da927e37757677c72629afc59c8a8124524f24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/8f54e1e4b65faa294e3f1be93e8166b8e39de612.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057787.63/warc/c400cab49405e9ac18bb43f0b5bb5df3b6a6a7df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/0ccb4da2106722578ba1ae1812ebb53e29b880c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/2b1ddbdf6883b41108fd672e17aaa82cae6c2d44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/4c211ca4dca94488ea92cf14499c5d433a04d141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/684bcd0184c0987dc969e48a7a77ec0ccc20ec53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/9fa8f945a42673af6662174b4891f5b5111878a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057796.87/warc/f080b1b4423aafeeaad1a9386b6d79cb3bb5503b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/067de232141def996cd295e4b0b5591bf6e97b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/3700e01d3270bba938af0498f7921febc11e4c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/3a60edbd1cca503fe7ec394ed812971c65f1a547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/4ff76d01642bcb5f4aeba4d4427e71fc6bcb85a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/92ff6eae30bdaa58bb2611e454a51e75f10fc0a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057830.70/warc/d5477b6f766d77a26ccd8f9ba6bff351faf29e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/18049bf9c7ecc477203b525c0ed054d41b780a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/2ba19927dcf3a853a9b2a8e07173dde5a9a632c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/5f82d97391964212455994f5dd65285f5349ec38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/670781408bcbc061c9b8c537e4b01de77ba8530c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/7b8f8b0a5d80d16adb3776a46fce448c0f642990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057857.27/warc/c0790d0f58204c86f51b3f1cab79723ed03278ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/1b7f6edf04eaccc2bc9ca64ed6ae7bd2a669880c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/2e9404636e637ab30b852bbba4b6205f9ae85d1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/9a0505379fc9c9b2248d32b9a6d543795a390edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/b20d2c217e373c2659b148b1e57cdb7de9f025bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/c527688a05054186e8ef422ccaec0f900c0d18b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/e2649794d63e59770426d015aa0b71f265c38353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057861.0/warc/f05e4f9f5c59018e8bb4f7f5025844be717df1b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/03a7ccd0602039b867bdd7a1b644f889b8993487.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/0dd3087160e365930bad7ba2446175357ae99332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/2b8dc26a234991de98347533756793a1af6cd73c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/622785e10c0d7f5401cc24dd7f893c0975152e86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/6b91e76abcff972ea59ae00808e9dde46c33b50e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/7fe9f6bbb0b13dde82c38ef69246d66392b323ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057882.56/warc/d03bd633ce8eec8a648c4c8e41c1861e3aca9ebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/464d232d04fc34a5ac0af67471c9a4bbb6a79cbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/76c340c773919bb6363d0c7efcb7ede38cf771de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/7c9d4b0af43a49e0cf08592f3b7351daf82e72aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/d1772e89e6629d6ac5c63051fdd8454f1ea7a473.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/d6ffb573d67852fe2d3377ab9b1579ace474b484.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057913.34/warc/e0640ae77790569a4e9828016f1601035d8a30ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/3a0479b75eddbb364d4e2ea3e2b41a3963bed611.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/6065e06b51a0a4c4833d889559b2e960871d85d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/715aaac25ca9d2099bcb8ca9cecdf48a550d5744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/a3499fc0a9699ee8ac8430d81aad79aa34f88485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/a384cbb2187cc5b0885b99d1e00803631a24573f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780057973.90/warc/da0e2866a46d40f78f98af21553bff9594aaa7dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/13eb56446830a2a539027df5690ab42c54beb10c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/460b41186fa7c569ad74c5574fcdbee30320ae53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/4c23e6e716c90b5bf51040c428d46dcf73f72b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/8ec1f5dc0b436f04231d13449c2a51bbb4dddced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/b081a2a7cf3be0451278f9beb3ed452912aa5377.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058222.43/warc/ba1baca51f38e77b40e6306e360667b7d822cc8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/0f2d0d8ab4d744bba27de3d2d72cf3534b9647b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/183f62dde237d96b520d8f14185139b9e385eead.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/546bdbb46cab7bc3f6d5a4c99ca969dad1fb0a49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/5fa055d89f025d9a445d9bb6fa72cc7f1b856f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/86c51e869519f201dd19672a0dfade565d94bd09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/a52f68abe5e266216421916e27a1eafca8deeb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058263.20/warc/da023b93170c55ebe5f1514d39e1f05e7a447abf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/0019d06c78c17cca1737739bfd9ae4f737b6e15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/0c9e824598e4c31a2c8af6562140276dc00f5818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/100999b6ebb84b4c34ad2b70637f17755aff3b68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/704261dfa4e08a3570ab2914b6d8c8752165c2a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/ad292238d52e5669078ad2ef987c3adc90fd0c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/d5d242dd61f148efc6294ca9cee90c88dc64b6e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058373.45/warc/e4def408bdd7af2ea48f5121cf7d3fe548328e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/056f08e39214646601f18ae7a65669a7d7542cf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/5d6868984963e61e0f5c8f8badb0dfa6e41fb0ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/5e23a9112abb499ae6cb14aafaa14585d4708ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/6e2a941005b1f0cfb5e0b2dc9b1d128ec93bdd6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/ab372da578b3969f7e51a64a4906c3fa57b000a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058415.93/warc/b32e60845a30009018de07c7e406642a0761a2ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/29e374dad4ddd654922144f4717b72e247c60ffb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/2d8c0cca3894131bc9a31611120a216ea23aedf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/559715357efcd748fd2b5e7d0d4b110edcd06126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/89cfc2076a5a6e5fcfa5f30cbc0a5db672fb27fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/c27acc363f43ef493c42a8033015a019b4f6e096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058450.44/warc/fa6215d53b85303c9a339dfa08abcbf3f4ac3efd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/095fef1c43a033c83f24073da7bcadd43b7e3af3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/33f29b4435909946ffac236719395af1fae015ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/99572809f7556ae4a6b0a3689d7b0fa62e9f6482.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/c01f2debade411e736e434e5c60c0860acc5bf13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/c7cc8017b3af74f7c565a54dc1ecf0a21ac50252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058456.86/warc/e1766a329676f76d0091a992a90f8fea8bb4d628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/2fd0158064726ebad248504734299e0bf974ce17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/456cef9d168f953e093af19751125636367fb2ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/4fed544916c18d3db80ecb2b843fd634ca828a86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/87a3ae83a28b408b9114064125240360cb548dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/90fc61639b6fe68cdd36ecb86ea7249037cc96e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/c08734d738b6072984d004ed1da6d0c92893d88f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058467.95/warc/e285dba4cca18cbf5164aa82d1060314920c7a5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/09f8c2af02d68b18686a0367f219c130ce88a606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/41f403d236f5e13e3f25799c4bcb535cce177b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/8ca9a8b65ffb670d74e92d3e7d50d2b345de0307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/9396772118553bf4aeb357212aa705a435e12268.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/b99696c9503e7bf44ea96eec0a07025e1be6127b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/c21957cb45beebf79821efaa886724c5329cc192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058552.54/warc/fb3522cbdfb3ca702f24d190afcfb09026fcf5fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/2e9af3f88604f905a7f0ffdd54f453002f62fa2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/33e6d0ab28f2d2a6b32200181222fb9882449ff2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/34a7f969ea6a80e7a4b631804b8732ce4a4a14c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/4b44528938bf37b200dfa727f1a1c177d3cfc1f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/a34ed6e749707d49c8f4ae7d848542e1379bba31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780058589.72/warc/f486e1f466047b955613f99e5c9e00b4b150f733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/4c99f23af98afb5a5d5c7f5cbcf6816ba38576bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/5f6f13d0f4e4fc9315db8277b888b7ee53c3ee62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/79a1a5ac523df24082a91a5102272368d5729d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/c270734a2342347a3b5a83af888d59803e3d8949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/d542aa47980e03227a767b74a1f65d023a789120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060201.9/warc/da95c050fb9a66d15c56ec7df90c94a7d9e12316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/331100e9ade69ec635baf72a5d830b7777165db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/6aaf007a1ae22c1804c545dfb18147bb3cfd7204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/931c1a8d18bf4ea694fc7d5d60ae8aa4b585b512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/d0b7fa7e672fe3dfc82e420047a77653df75b5b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/e0dd98b8587d5d5bfd22f094ad92d3da789be46a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060538.11/warc/f4f806ad4d65d00ef505d722adccd0075c4a0b31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/01de67b6bff5ae8f176d0db3f6b7c6784a17eee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/1002f0c3972f82721be577a2c2f0d52b08b67a20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/8f25c4bb7ecb5dcaf075db44475219eb72377bdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/981f4398a75746651f73c19cd6485a5df6e06fcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/a1e312a9df1c5461d957dc2737ca4d78e996217d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/c657ffeed33f376a6447c63df3394fae9e106205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060677.55/warc/c9408056e2545638b5ec8a357e7bb2d101f9dc0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/03abb20a1d9436ad2fca6f0383959245a7d178ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/1871cc9b1aef710547540c9f761b674c23f64d8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/1e7ef92ee0f2c4dcd911b1a652bb40038fc50839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/20861c01aca485692b8996dcb377ae219b8b61af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/34152e009b4fe0ce8b357bb4fa35ed124631d49e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/8eb358e5071b9b5ac2425046f3698606e61aecee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060803.2/warc/ebf31a399ef29765f956a59e17f537cee214d7cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/4be4f7d1bfe2ac521f320be915968ff973b7d524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/7ff17026e4354304468590279a59143c3ac60302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/827dba22b96f73e58ba18e96c12809869551cbb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/b848de268f79ece7398e6ca74b8472a1b2558567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/dc4416fe52933e2aa2a83a98ab90f68b570cf8ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060877.21/warc/f41a18f551460c2398896ca2bd07f16d534d6c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/1810428ab8dc4f293123955af91052e10b80b45a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/20e6c44bab2112c047df815b068b4c99051c5cdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/5044d73b4d7899058825a39b150ef7b23f3ff202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/51a6910ad79bac3fee9b392c20de025945a40fe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/627ce18421180302148a7a50267aba6d7ba205cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060882.17/warc/d114d04d910aa8a18361068b77cbcdfef93a94d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/17b68d84176a7c7833c2b69df634171934ab884a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/28db1e93753f37d70b590299d6f4625459adaee5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/5eff17ca45b78cfa8c3c349b74141c22629a8ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/64326d4787c3ac1247e82ec9228fab78796b6c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/a699c4ea102601aca3b619ffd1a71198825afcda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780060908.47/warc/b55800b6e251cd1a170ea1ca6ad13d5b7b26ac27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/14c9c9171e1b6b78916e4a12b984c9af2dd88a53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/35ef8029c39035a5870a5644fbb75f180b931c7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/4acd87ff611950edd4a29a51c3084bb9642b64b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/7048c15297738c61025823530abbbd2dbf6b64f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/902c12402bd005ecaa0437c4150df07ba3c285cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/d6f6fcc5cce1ef545fe67b0dec5e36c4a61b844c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1631780061350.42/warc/df749b132642d2e3896c1564b4534a276989eb69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1717a748735482e2b58167d26abf12785527de87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/17356f20d1551c697d6dd8377303cf4ea3a6bfdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/18af55f86884c1db7b06a5dfb9e3384c1c468faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1b1549f25221ac2f005f948b4ec44448fefdb52d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1d0809b2e46a39073682c8cb92de0e7ab24e444e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/1d50f74db1037ae6e2be58205fa4e3aa71a1709d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/201401392f602b5f45c84f5c03973e8ff58ce6e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/2d5e070d4a31e29a0b0096315887e7e039b2e153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/32ddaecd272d3105041ac86b90f820f5d2f34f52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/33e94c37be71afe10f70c8a39e152a2258f43e84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/385303090f2c754f0857cd9051ae846c64f7b15b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/3b3b4f401eccdc2187d86a0647b96e648e870a3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/3b4671a7d535500f9a877e0cda5aaf0bc989219c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/432ad93a7460a8e0c8e5b1b4064f03a96e1a89a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/44b5ad165e779eee4f6552058864ad2586634fb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/456fda5258bed77854bf8314486fef102b5688a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/4bbb6904461d9830e116aa3d65d52397fb3f5e27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/560ba4ca415d8f5d08f614fbba9ac7d50c5eecdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/5c012df252dba420cff5460c970ed43037f8ad71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/5c3b5a02d16472820338db0bf82bcd789795f4a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/616662e94071b96d4d39ca6f444ba574e3eab2a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/64be0b3740a5f900adcca3d8a068c8faab8068c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/6578cef55728da948aaf4c595a759226ded2087c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/6a5a9266c41b4037703b5e51233202b98478dc83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/72f1da6c482cfbc3c6860b517b0e16e3ed2eb74e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/74b8c676db98bcee4bd483c3122ae986a9511d5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/74d7472588fb591e6db7523ade891aa724f33754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/77e298a073afffa98a5e826ea40f45425db6dd1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/78984b17e136cedb8618519e6035f0ce1fc32c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/790c349da914fc2f00c41441a8d88de4caaedca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/7bbd82c8177780a9d2851868c5630c18ce7eb94d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/7c0a71e7f1a6f9a2402ec8440b49270854576775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/82063e7148921164c5928bee18c8768e9b7780cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/820a41303d80ed145b1291d52b73ac880a28ab4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/91aaeecd722f444755524b68fba56b405173b9bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/92685cf839ab534d756d0abbbb3f5b57fa72761a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/92e7558f13e21358611960292ad2788f4f0490ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/950fe2d3e74694927a51911b6da7da1c0061b01f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/99181c1c1f1c76ecff66a2de96070d36f9234961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/a030d4a15693337f3e2a01b41b705e192a0a064c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/a3a8065fcf40b54a36b69de21b7c311afa7e640c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/a86247dc486d7a25116ce3ba7fb69525e8dc515a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/a938ed7ed6a058539af0a6c267d023c6dbee3567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/ad6291d9f3e91398d79c6ed1b079d64cbd485b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/b1a6cb8935439c38c5bfd3b1b1e108ce1b24cb33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/b3d547104da31dd7eeeea350d8d39e96801939e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c2e0b8e5c5c7406a4eebbd53b17c66cbde5b8062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c3ba12ec2d359e7c3c3d6ad58cf72f3dcd8020da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c3f2bca4846bbcbc9e83d52c042d9d560c269f92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c43f1a5cd291bbe87056e31844e433cc5d8d49b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c7132f9490e00fbfdf65dafde97e19a197514fbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c8d638567b6f1102b1532c9efd32476217c953f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/c9914717556ea1d31b09a1018f1cb74159ed3405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/cbafd25d9a45e971382671fededc9f38f4d50fb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/cbf8cd038374b30a16ba9a18ebfcb100c7ea6b40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/cc16cbb35df2bd4828a991ff480dbd554fff13be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/ceb78ad5b1646d12287235ac1cc63756dc30d77c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/d8f2a490a54dfaf8ff7f280c305030ea53b5fce0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/d932181be366f319b38396e9e750f176d342d4f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/d9fe122115e7898627c2d6e2632d6b9fffed8abd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/db303cc9bb096adc02a07bd23031662fae5cb06b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/e1ce279495be9d5699695fb6762b27e274de11e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/e35a080bc2155b2ee8562a08188f85ff3a71fa41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/ed19c2f950e3dc0d32242d1edf87aab2edfdc589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/ee0b6eab782bec1a797015a7be9472442472ed90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/ee324f0ad22b9e9aff86853df8eaeda821bb55de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/f13484f61e68e083d1d723141f56d1135cdd61e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/f4def36b41cc38ed61c1d8abdb745bae201db612.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/f599ed6f288e49fa0a9911769aa272a68c4bf8d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/fb6c9afe19720669d55aaed3a6c97c185a551ceb.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-39 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + +work_dir: + input: "${oc.env:HOME}/status/cccc-CC-MAIN-2021-39/mix/input" + output: "${oc.env:HOME}/status/cccc-CC-MAIN-2021-39/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2021-43.yaml b/configs/cccc/mixer/CC-MAIN-2021-43.yaml new file mode 100644 index 00000000..9e7d30a3 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2021-43.yaml @@ -0,0 +1,834 @@ + +streams: + - name: cccc-CC-MAIN-2021-43 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/06cb3e3de15464866db449e4e8a5f8ebbc640463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/0870fa4e3ac433c83df7537b4342b7191527f741.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/092422251433b03d8375327d96fc6da70f03f8d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/096ca2ca97bd61076c814c8d2800f147164ba59d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1151a7751d7a8ceb3c0a09638f27ea050154417f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/142b33fcb99ee8efb19501cea5b5c32459934e9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/0181f3bb97eeed5042f23b1bae019a8d1b342364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/30557990bd22464b006f33f76d4a8107e36458af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/478b5385e761aba4ec923938435e68a4dcdbbee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/6ebabd3f265b8cbf9b41456e428ea13654cecfaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/72a31a286ce3320877e2a56ddbecfe20aec6baae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/c1d55a33d416b14bc3722df942e2a2634a5190e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583083.92/warc/fd99c426c24e430287971a31ee0827eca94da91f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/06d47603bcde94d4d34b289d1d87d38f9cdeee77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/70b94692806eda9faa156c54aa1da4a8f2f47c51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/8623173361938660f5f8de3e122a01e30c4b1981.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/d5892a35f852ddb6b9ec5a1e7cf47a14df3d00fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/e6a8047cedce51e8ec013f6b8f94a0549f9346a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583087.95/warc/ff94e18a619f2afdb6235a680ca943a6fd65a79f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/14fc4c5e3fdca7164d206c0cc1a0112d47e09b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/1f373ff81d95b16f3f2c8823f7f41374fe819528.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/2245e1662ff8206e6371a132e93fc8d3b7543f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/57c736b595b02816b600466e69a9290ceb1d0479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/65eac39ebc0989ab995a6469bf274528d29bfa16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583408.93/warc/b1aa08f957d44ab62386da1f1b0523b2c1900c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/379c9898e5410aaec4226a5b9691f7286e6b07ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/55fa87649a6c0d46bcd9d671102facb1231d954d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/855f49b2ee0fe4575848d28e8ac1e043811bb683.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/93fe5de0be60d80e54eb3df969c6375a524effb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/b75dc57f061078981121c075fb7e1a8709a1e84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323583423.96/warc/fb2c994c000647bb7f59b070b93b9ca78900d437.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/08408ed68cdc411e80a1af61c2c2d353bfd46212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/31c8b99fd9dd4b60cd0791fc428baa5e4e7fac5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/4a100acb0cc709c99efb3cb69e161ca93632ae65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/593eee75e69fe8b411c27383aea8969cee978f79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/e185aabcb991ebca714a1807ae3d8bef6bc0e14c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/f0448db62b4e2ce9e4846e771672d27c50865c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584554.98/warc/f8cd7b7cd33dc624af9a5e68c61c8f4a5e2a20af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/4262adb448ea180f06a539af19db0efe86f0aef1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/4d5be358387fecf160cade8c5e1f8c24db893897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/62692d7c34e74caa4987f875e26bfb8e8e2cf253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/a2a4cb172f18b7c1f944d3464383a31b37ef8dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/d448a2e0d79dfc2a23fbdd86d42a0f030850e520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/fad96c57679261267bf8b020237a0c3683e69bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584567.81/warc/ffad16f0304d89f07363cb7d2eb4faa382788ebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/117e44ffd853e7da547b99a0b571a1edabe5ecbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/18e0a08d3f596181afe3298bbf9b90afeeb508c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/25ce363c36f818b29cb71e79bcd35a8a64cb9bd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/3a5d349c76e5646aaeca1bfff7bce1af6407eca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/b3118a1e42e5e4e639046ec36659b958d8410ae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584886.5/warc/de57a881530dec3692e1b327eaaac4d337b27f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/3d6e2b413cfbe03dbfc25a76b7454da85ab5aa8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/54cde84409fa134cce18789cc4d12a791277e7d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/9ee633e534d06f73d6cf0cf60df2bad96f6f7078.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/a413b944731d3577cf45ce2797102b602218a610.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/bfcbf9ffa3b2a9b36784191683d77ec449798be1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323584913.24/warc/c860cb572d9e3e3720d6d5424b9fdf1f6537bc0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/03d45349635c1501071b57f83b533b6c02cb13b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/6eb4b37336660929bb790090ce3d72635645bede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/7f4faf0072e7db35f897f8d25a73d1b78b88c087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/c065420e657cf5c23323ce1b4199fb78f544f89b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/c66c02bbc71803318ad3010b6903b2ce3ca90f44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585025.23/warc/c91da427a0f212526938a4081dbba28c2119128e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/0ef317a36a43f148084bbfacd944609be94dd959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/19adab7e1aebd10554ed1d62c1b6f37debec598b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/71b0ce66ba70d3798b5a66ad026563e8679a3ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/758d47cbc44b1a56e0916dece8fda927e7993416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/7977883734290f09b9bc4efec93cb607e1c337ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/971383b7741ec415316bd022ee8acebf20aecb19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585045.2/warc/9eec5be4371a9c79ead6b233ec2e8e4986477004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/31d3c25407a4c2ef48a4585993c14504c0cc9b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/367bb17cab83b1acd13c24cfb7d3731118abd34b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/4d2b75d2566c941c7808f418721108c71ac4e426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/7ba5e129042465707ceb60f2314711218457d27b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/d58d144ba1706a96269aa17d657eb8194ddb663c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/e0a0cad87398254dd37aebf8506d83fa4ef5c5c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585120.89/warc/f7af0abe1370f7c480b8c7b4b9a1eb47757630bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/005d8c4c641b362a9d526ec308e15d3faae8d983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/4b661742b52ea30b692057a23119584541957c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/55f4b3159f596ad0291e9f7a4ba4e264bb60e264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/ab0884bf23617e0198f9b0c3aa6edbd6145cb539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/d8a45485b8ecb4877b1b4672363129150735e20a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585121.30/warc/ee40e7d90fe4a095ad1351314183e5d1d83796c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/1d04eb159b9478d6ae42bbc0a07d64cd3175182d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/21857c684694a63a5507ae20f0593f03bee5ae29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/57fc7aeeff7fc0eea77cfcae1e3abb25409664a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/b6e6a335536639447780b3fcf037e8959f45107e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/c3689aab479f225028e5421d086c55665b814399.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585171.16/warc/d99258baa9eee50da81ee18265dc5bbb26d2737d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/26e0914ef54fc670888ea5014c9a940b3f3bda80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/51d39cd310223cec19726f2ab474fe16d96c0a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/681b36d30c05f315d269d33116c31b863c2da30b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/7d140651f45db6072b36a45701509f19a01f5b92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/a24ab4f3c3629dc0a3306e8f55fbd0ccae63ff11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585177.11/warc/f3782dbb276d3689dd48a130462b863fd96b0dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/15556bb38ef198b3e764ae9e690565c5b52cb166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/4967c5596bc618452fefe94e6fa87e80f0a2c872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/87f292fe979eab23e97f816a657e48341196dfb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/c0b11433f71c6b5105209cb71a2e1c8729060f83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/f352584687bf00a7d6cff042c08e54f5ef7a2d0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/fc61bdc0d6c821bb253a555ea03165272452d041.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585178.60/warc/ff64b6aa4c4a063a24373ba90db70651d19b28db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/1341ccaa0e02e80bb286ddc9d326549b80742bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/2237969ef0aa864211fe50d96e3e4766201dc133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/6afa7a04ca84be32287f4dc8fa7fbad2ca1ba518.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/7167ad48ed481f045b1c4ee2cb0cb67e6a05cfc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/89e21a46fd45fccbc155037b762b8d0dffc87e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/99e2cf95a1555632e7e74fa86fa96776655a35a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585181.6/warc/fa30518ef19c6058b841791eaacb340be679a0ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/1c5371ae5ac2e6bf78da2e00e7a2a5b9f4f6689e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/231ed2f8fed29e9d828555c335e7416386c3b8fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/5d2d6301db6a32eba698eecdab662ab5b2caf31b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/5dcd0a46c9af07321fb531335c53ce259dee4709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/c3c0241583920ad9440f7f3c5ae1579eed8e1036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585183.47/warc/ca413d794ccf02230f4db78c3899dc8dc7dd4c32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/26fe8e02bb82fda6ca0477b2ef2aebcfd5c8255e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/413cdc7ea7784868bc66489746eac97379cbe681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/5a18567ad067af024b54ddd806f5a807a4ee4642.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/b07edca4fe0937a3de6834300ae946fa1e18670e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/cc4efa4642ec8aa726e6b95a27d86a5aa2d429e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585186.33/warc/cff01613198f5a99d5b46203cf7164c99f0ea1c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/0606c0c94cd1f8110dfcfc3fbf7445b8824e1cd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/06522ae793d4302823c307bf5441b4aa4d5e509b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/3f6a66ec6f6f4d02873aaf43c8b812a245579f32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/4501c06d30f10809d2b37be17bbb1d99335f26b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/45267c8f76af8dd871886446d2dcde45ca6f9530.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585196.73/warc/4da032e8d8ead00edfbfb2ffd49e6468852d9660.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/1065d7081134660c9442ff897528312d415faf04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/1fc6f89555d4e98ceb3f46a530173baa61cea3b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/2149f5d8519de9ec080200526d4b5794da54389d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/4c6569fbbecc168c55c82f2a37f7603fe50963fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/7ab5029bf0a8389bb0abbd0a9c65a850f8d58d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/97bc01d762cfe7907c46c477050051ec909a5848.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585199.76/warc/f2005a334d931cc40a8f31369fd8d4ff1a01039c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/0d133a5f6817218448f4c308924caed6db57f98a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/0fa46e615d5064d936531591c86e6e9d1d9daabd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/5e7ed16890aa016bc4bb3d794fcb4601fb589de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/7f4940edf398ff4649b693f7f889da62e5e94e25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/b51d31ca4ca6a652ef48fcffe32646251f2ae371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/cd23a00526267e6a69bcaa7bb61a0aaed7fcc990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585201.94/warc/dace5b4664a6b269ed154331c8aa454d4e624df5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/44d4db959c3954da034dbdaa01677349c0ab1cf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/709115c40da2afc3c6f3fb897ad01984a94bf979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/9459613073cf73892e7565c8161910a8680bd6bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/9a4b9159de11896fef485fc24250f8aa7be9aec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/ab77703b5ae3c637689f9c8271fd869ea890357d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585203.61/warc/d70cc3dc16daa122378c47397b5afd7398e5d3bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/47e4215fcab74fe603ec96f7881ec8ba2b296380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/4fd52660bc58496d7ba862aa37a9a533f1f8cb2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/973978848dcf7f77ce2a0ad82aad5b3f2e274e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/a683dd24d655a1608511b84fe211488bbbf9a4a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/b20985969cd92f10e8ffdb0121f0ae9851040bf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585204.68/warc/be7b375f5f1cbcb495368b2de05b0135b5b83cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/11af36ef2f6c5ec2736a3f6189b0917809156ce7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/12cbfb7178970f6af3d8cdc8c34378d36660523e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/4138ddda31bf60538e9f525a7602c733dfecf815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/742b99c3714cde032313fe7ec31ae7045721d029.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/96e997ee5c7818efaf8dad624eeb7927154a9f59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585209.43/warc/c793585fc063c2bfa76a71cd7886ae3c45e79c68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/1d0bb4f2ddba4e76b0b4071f96c47052299eba9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/25ddbeda35a7f57d8136975e1d0f6e284a2fb323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/4dd40942a73f26691dbe5c001ac30fbe5196c61b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/730c167f93fbe08757dd616f19d5f508ffd3c38e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/9d2971c61af98a1fd5006836050014d2a6bf5e52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/bb8280cb8a231517a09cd5384daaf3c29834721a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585215.14/warc/f2502cfc3bbee5e837ebf49e003e1f530b5d1f0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/2c566fafe0eaab169d8d3f8fc9881f05e93d13d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/405b10af61d4d6a01f667281808f97380e9d92d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/4d5abc9136a48948e46b4ec76ec7ab8e9e11bfa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/7d64d18a4cfd9686594e707c037d935d3c2ab616.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/b1d11aa31e0fd78f974fae0ebae1a1c26e2200e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/bc9ac97e0ba2fd19a407e4a2dfc3067598a337ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585231.62/warc/e9621b9eb19d1a00aa17b44ff8cead3e4950c146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/00c7890ebdb183f7dd42fe8d3d4a1712fc043fc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/37c3ed8a8c9d5b3b19824faf3ca37adee70b0089.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/5fc05278ea242c99a6ca5cefa8b74c62b4b94351.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/b6bcc433e1f285288d3471a4aafba59937f7fb92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/bdffe6cb07f2c955fe63beec7193e9f32427e366.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585242.44/warc/d3ca38dd8690a2d38f9812258f2806ecc2b82919.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/0bc12e3f232e48364611d8ab5e416267baa09699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/3d48942e3f5f90cd5e9b2030b30249f9c8597670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/455bb39312efbb42c4f118f4cd10eb5fb764f75c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/6aedc7eadb36572f5f48f6983f1ade8ef88eea41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/8b5f9bd7a6a3e14c51d81434bae4fbd6977f97f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585246.50/warc/c0b20465a82f73141e2024cb64adcc936b46def1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/9b2184ee494588afdc124609acc0fa7731719762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/ae5162691fac8fcd438cc47a5e14842f573b0374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/d0e7fcf6d25509591795ff8c1c855dc6f9c42ba1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/da1992493330e1af6de1ae1afc081d124779cb49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/e6622ea75d666cc11a0b91b87f9cab2d7a5db0fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585265.67/warc/f620ccd39d35a7ea805186acf730baf2fd7e20b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/067f504188cf0bb4907e997e1b09585ae4e4858d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/188e76e2f74bdf69c40f9919a84d78fa8ecb81e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/1cb48c6cb6720363fadd2d5df762b6b59f63737f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/592c8806653d2ea7462b04ed8b498dc64020b4de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/5b3346c5b58f4d3fd66b385d21bbd8868e32c980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/611357c42f87eaa7e91588524cc999f1c56324ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585270.40/warc/8bc2d090dcf49fe811b8626c6148b37987646b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/168d1c7adbfc7d8f1b0ba3d2d6de6534392de6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/76d19da4596f85a23319ed9baf78f26c3f991768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/97756e8121e392b93e7d7e929ddb48105eaf4768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/a609c9ec09f869ab630cb18388db9e01e1a26952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/ca43cac328799608e4a3ee32a7d72337badbc1c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/dffde2b04e8ecf0fa79a6e2ac2d74b1c30e3183f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585280.84/warc/f52cd524211c3eb5a319bf941c1bfc144d04b25a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/3cdc22cf59f038245b1d19c2a980679466ca1ee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/7589d69380cf869762fe41c7722584bd249e3447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/846d7bf13a2f82518ce1108f7b0327fc56edfea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/a606370a6ae460b2d5b6df0ee1b6c6382f7b2647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/b8fa6d77d12ff483e45890ca58382154ca852023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585281.35/warc/fe170003433622ac346eecaab468ae6a036d0438.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/05167d22c5191466a29818d28b8b55164d1fdf08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/28767e272b3196127d3e05cbc50c7425bbc3d73e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/2adc9f79f01384b15bae0c9d6088cfd48718096b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/526886fbdbf7222f3b005f16c3e79fc4e3c84092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/60572bcd9774831ee3ccd4fa3a72f20da16c8102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585290.83/warc/d4e087b3a82606e4dd2509aac80f87c35e9f6407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/1c9764fab2ef070eae21fa9744010a21bed1d79d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/2b652b821d4a9ab6976b573104e40e330fa03625.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/4a4eb240a85ede9d0bcdbd99b38b0509dba410c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/503919eb59ec1dbb2f43d31e3a5bf0bd02df8f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/50aaaac0c7e7fe66f6a4979b18ba873ce98d474c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.56/warc/8980970774a94c9c6e9e92ac03042e8c8af2b8df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/07718b06e9b5591a4c875eaf4290f6971aa80c1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/1ed02d3bb5eeb8512afb368ae648f84147075dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/3557c8261ca3993b4c00e50a352d9b1bdcc16da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/6ba086dae9c588ed76687f3e4bad2dddf170abce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/b2dc26927a7060e9c852f69da380cfa0a8525dc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/c889a163170fd8091093583493fbab7072146f7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585302.91/warc/d6c89a1633341a81c58062c03a217b6c39cca78a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/04f1413ce85b798a4fcf21a7d96ed7e3faaf6747.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/37036d69176a2dce54a0d3d940ac4aceffbc309f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/68aaf56d296c790939779fc24f081e97d2b23fc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/7a49c00ecae29ca58f50062950218d3b8778f9c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/8ac771cbfdb4052b18a4fc8b62ba3329fafb073d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/e52b1f541d500eb55f92ba7083de32ffe63f0ce1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585305.53/warc/f50aca8b4aca409d86f734c7f092fd0a172b42ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/063aa4df521f8eefde99ca5edac9c89bd2348cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/2013b9ab053377fe9f5a5b246c4c92592b57aac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/ae1afee7fa86dd6ab5fe9f858621e586c1c59645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/c115acb113d3cf15a2acce33db9bed64ca401537.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/c6aa790ff8a344d8f3a6ef96ed546cdcc9039cb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585321.65/warc/d36bddf1959812f47729903f7457fb8f8784fcf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/33a1626fb98325d954badd743956100e222a8a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/858dfe74a785f88663f4815c7e9d47e13854c980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/c2b42b5035ee00e3a1418822106ec33efa2a66cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/c9bc763e6285a57f8e2e8266b4af6acaa67e9faa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/de26bfef7627abf13b6a358f656113e0e4c7cb61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585322.63/warc/ee97b27c537db83ca80e315faba951e9053fc1d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/32f4e59a5274f64dfa5450687962fc04a86f4f7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/6f8007f6456d17f1db2c8981fbe180fe52d923be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/b0a8ec7ce694358d55fe2f39598552791c4bb088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/c5e2cbe39bdcda36ce884548d729e400e55a47fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/de12b851c87740f5f69419ac633e92d42f4f3dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585348.66/warc/e57d39cb1ac3efef4a2abaf8e09c36f80950057c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/15f23925e0d1eafdd584b3b795b80cc7becf0b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/1b046ea5a8a13ceab762032cbca6bb21c8265365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/3b4369a92f60aab2054f502a289073bee71c34cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/779b041d6d4a07473f65dc66d02bf83e26ff4812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/d15cef5947c09ae4475fc330b3b053ad3cd8dc0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/e010527a539adb55cf04a0296e6977113dbaaded.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585353.52/warc/f910ff9da71e2ec7701335533988f8d4ab0a28aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/337226c753ba8d370397f490d5dc4ca35396bb80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/5a598ecbfe0fbb6c4169e8d7a0ac53ca2d440638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/6fa25621b0eadd607926d9c9a18f49f9e4052608.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/7bb21140d6e49edfe220530e366a1f4d8af4b20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/a39c8c739f45d5b67ae4a3a8f7cfcd23aaf32b94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/ac87c5ca609cd01628b1c5f2f3541a8fb487def7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585380.70/warc/d33367427ed6533cf7729f948db0a7e8c7aef118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/180faceeaf5b7cadc2832c2e24341d76deec6082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/1af3932900b35b222fe2028b24e6d54eb611755d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/201160e32b4df54a415c120a05bd5e21635e65d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/2234bb1c318c314e9df9ed9aae118778e24e156a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/86a577446197a9e2ba66a37b4b995cbc5d0845fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585381.88/warc/e224b32a6ddf01fda893405f5d1545a504da39b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/218afef50f516de65768b09144af91728ad3c27f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/4a01123b50b47cc1b587796bc2ec03100c34f0c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/4c09df331c3c277d4fc69fa0801f0ec2173e6bb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/5c4198eacc22c1694e198e0f6c7b765a823b96e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/a3b04d951ab24d385cab86ec60058b1926958aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585382.32/warc/d0afc436774b40d393252fe1b95f72761110bae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/12427090fdba331915377a31e2f7c7ec4c212306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/24105b6049259244af6232a5f134867a0da8860e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/84ed0e240755be056583cd00a4b1f9ac5d74b57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/a1e4bc0ea9bcc9ca3547dacaaea2ef5ab58cd72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/b3388f20971d547824e47765c44a6ca5062a173a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585405.74/warc/bcdba2134aaf4bde20de7064ea948b9bef1016fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/3a55a8dd90e43ed0183aad4a34acdcf4e0c928a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/3e01350eeff18e3cb3117f88333aa6923b6b035c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/77f7ec5ee9d6e1d6b535c1b48709c191b3591707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/7b1a385fb23d3c0605a039f9572955a67f4b1113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/87b66f25a848cc75f72f569e418ba4a2af4d27f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/90e0890411852184eb13f270295f75dbf36490b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585424.97/warc/a2225fcecffcf54db949e0e5a870b05519f84dbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/2ad21d1dd942cb5ffc5daa16488473e4462933a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/502dc06c3d60426f971a0914c8cb500ed96461c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/8dd590dad630986ca100390a7a0293b17696e91a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/aa92ce9fbc51b824da8051758c411179e2e14d7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/b5ad07b78c5f317035b72456061ef29ae608e163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/da981cd227bd19043d92341851c2b30bc0c7d09a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585439.59/warc/e4ea5a42a7953eaae729ac30be9fae25be30d036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/0c9203d6d6bfea58c52ed0727125282b3fbaf836.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/0cf1bf1277449392066d4148d5e4e4fb2a37dc5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/2e16b8f39e95043072ce8e52c04726f754b5e204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/90cebd3686cbd653de6ed00513ef3bbee45dae22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/b5fe91f90b39d576fda8e6dec30848110630a023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585441.99/warc/f3af82246f90319bf15d3b72bf5fd6e819f960e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/2c024c39bbb4857954fcaf6ca64bdbc48d079100.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/3f431e1fbbfe64ba5a5887e4ca7de630e56d4065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/4e4b8e6d2915bbc6a285174df8adfff89cca28ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/5ff6cbc843c9841e9d4bd0c9a47f9fcddee011b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/63e8fea66cd18e937c2db684702d983e9a893684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585449.31/warc/7f0fd497050e221d219ba43ef9ea9e95da4a9cb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/2a47fa50568fbd60a4a52bf719e21b93a70598f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/2f61166aef294dc97e8dba5ee4f3946574a5626e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/66292d4f3d6ccf2ceab7866ff6d7e184eb5d04b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/67f57b0a205546a8e1c70b407e82990cfbc6e784.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/84f38c10eefb3e8d6acb99ebabe1f78aa4a731e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585450.39/warc/f50b43298c6d8c1446f0bb2b5421bace2ad244db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/129d892883c2fbd225eb1be5f3f50847108a60fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/38bf51101160b92c42b4b663b1fbc131a2062af9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/3bf1798a6fed45175d156907fced5ad5ce15e24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/3c1757d9238de29de21d8ddd1ea5cc2dfaa41ae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/530e07e79758ec3da5e8bd6a6abf1e699da4bf1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/761b9dd928b1df1f6aa883093057a8ffd86676f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585460.87/warc/769f89c5a08be28fbb583d1b1bece6fd5747df79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/1865d9c838531d06715ace2ed7190298324e407c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/2a1d8bcb881d77a12e26787b15edb430b4970a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/2e3697bdd53b5e72c6159bc659e5eb1cd922a5b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/5543aba806b4a4396055def858c02291b0b80b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/81cdcb3e4b67c2fcad3df1ae0455c9bcf196ec8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/84de1d59d6730bbb580471693fc24d904972f706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585504.90/warc/907aa4daa7a3ad6656adebaaa988dac99e1c8dce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/3f19e0d0d0cc0264598f6750be9339aaea9ee529.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/8446a39ae84a6c85e365d8eea78a14daac49e5f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/acd2996365f546a0d95752aa65c346c04b2d4302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/ca802c21a76d01dd868e73cda79c440fa1cd4f58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/ea4e2ba15d336f146d90f37e876224b5dc27aa70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585507.26/warc/fd9fd7855d8f1c2b192e90a75829376f8efa34d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/2391cbec1ebadc4a762433e5092a9e24e8c49db0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/5d2a74fa135cda1d6670077fca1fe65739d21186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/68eb2637fdb465ae1a7db336698611633ce60c2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/79d2709c2d4e1d59b106476fc3b2b8b45b19b2bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/8e9ae89bb437a232e1899178d166e723b7bc438c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585516.51/warc/dc81df472df254316364b957dbf1af5e5f3abcfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/04ee028702abdfe1b80655edf0215e0422838d8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/1fe88620cff66ec443145556742a59e865098737.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/5c783c5a7ce9487b6427637100e713cf41788b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/67907ea801894b602bf23c3d7b1c5feb127c9ee6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/84c6d4d3bf6103dbc35998013a4ca2394421b534.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585518.54/warc/d86fd7c5036386af10e2041675230280394f9a34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/108f02e87b1a1734c88d13288bd4609792aa1c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/387c23d36bda7217fccf759f34a421d018440547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/43a8c7268ad3db8834aab7dc9617417aaf08ca7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/6e159f2da439e83790f0ba9d6b8bc24266fe705b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/7f3360470a31f96e713cb4503d5b3a67306c8166.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/b52737d156a53b46499a13a6af6030b181cdd3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585522.78/warc/dcb382d360da9ad3bb106d315ac3d3a234e810bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/1f38dfbc88438adfdfde78dbe4a460bdf18c6336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/59cd2b3a7a95a58ad5c3d874f43e67a0b0470973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/6a3203163a37c16474827e5141ed4af852e963bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/70f131d58d5408ca2999214320f96d5b9bb453b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/88777eb66faea3eead1185792a4365fea1a5a728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/caf9866468b841501821d824d60e9da59c446e1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585537.28/warc/d6f1bd2d215111396690a0e58d2f97e10bae49f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/02db1590bf8da2f9ace948a820f9000f1af02384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/178960ae09949515d6be50c61e28a58328749722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/2be439fa94ac0bbe84a0234a10de87d8cafbf1d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/53ee88c1c913555f63c1778d1c32c1de3dd35080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/6bbf6d880448b3cf2ff4d79650cb710b436044cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585561.4/warc/f71728557426ab34f46a9b72705985cbeeb7dc14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/4fad932e4e6bbbea800cb04196dab94ff83ba37c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/79cb46aef34cafbffc1640c084532abd01acc451.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/843a26b1492e1bfe413bef4f0d9aa869dd501bd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/8848af6309f84d19f9590eab7f86d6547202d326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/92e0bb798950c5bb25726ef53fb07a5ecafd910a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585653.49/warc/acd373a1d79f0690cf9cf2206c03234bb7017f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/08f163d0530423617548fcc487c3cf538b8b7fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/760a6129ad58a2c74d7ac992f73213b1b872fdd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/7b558018c9bcc185a89842cddd446d724b06ef78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/a2c619f1b25eba6a771508fa4a4820353566b278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/b2440863b01f540d50b5e8d36067020f344eb2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585671.36/warc/e924fdb11584673c12eb49d9df66f6277a62f2e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/411d3af4ed50c079d64e1ebf50e8003dc5454dfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/4b8f1d2e4f7adc936af0f32d6904746f3738b589.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/8f1e525d088d824a277aa0a618120b25b1931d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/91a851017584d6a2fb2868020cedc6598537f168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/aab8907ca598250f90daeff51149f9eef7b081be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/bb6089d67a8504d0cd99674d13bf057cc6a8448f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585696.21/warc/d2e0130eb2ab28a07582bbd765b011febbf88620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/1b4756d79b0f7e05682fa7a2dbb4e0b56d4f0126.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/2addbf1d769e35ad83e5797f58cf0f1862d6f2fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/41084e12cdc4163e56f64d672bd14b11ca9102f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/67de7a5260dbacb81052ad61e42bec9ccd014129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/804de3e1478ea537b1c6ec6390b07125e69cbf6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/c42bfdb95df087eaf367b72060b8a41ee92446de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585737.45/warc/d52507777cd71252d334b51a494f51546f3e87a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/25e604b73aa643c4c95aebd4a54bce085acfa16d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/442ef6af6fdb9a550f3cb95195e97a74148a1494.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/537d8b1d12ee8345e0b4e55d4945d19a0c443316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/b45fd915b1f6901b800d8dbb405d7c7de8c163d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/e9b1f0f4d815483b9ee180cf75ddf34079cb6d6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585768.3/warc/ec47dff78c118420bb908a8ec8c18147aa4089bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/65fc7fc95d9d6122c67de33fce8a212ed21d2b6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/b0410e4d2c10e033e74079b0bcae8e94a51c7891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/bddf0a5c7d1e74ca7e267d2219cbc7b583d37434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/c38a28b3282f8389ea228ce5b2003baedd39cd7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/c4dc130124821d2a86827b193563e08cd0725d8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585828.15/warc/d49a0038166ea8d5f67dcf24cbd18d1f34581c0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/12ffaaf1bd7f3a269097880789aa4f394d1ea6bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/13c15c39d2b670d33301053e71e32e7d153edcc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/44bd886b1b4331a152e7483bff8ec782894b5714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/4d5fdf507106d540e387c29bdbb45bc5071659e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/4eb2bdeb64be71b4acfd97bc865ce440246b39e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585837.82/warc/a249cf4f6b5279f56586c81b78ea5d2c96391963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/3a030fe0e8b814dc83754bea036d58134fe88025.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/73efce11404a286fad861986a77476782c3e4aa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/81ea29e414fa93d2cce69baec8b1191ed025c23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/97b73089aa491d6ece443747ad8e210b3339dea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/a1a02e2457de25026fed1181c1e76bb235f43af6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/b6dee61e9abed36a10bcca71c57ad964c6497a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585911.17/warc/f903b229dd961d0e9780b304877fd4b4a4c1b2ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/3dbfa90cb20642d34ea237dd75d358c987fe5fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/4e6914750ec2354e47f55aee5bb248576f12ecf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/63d78204e8654196e7830f6b0a4572ee443b47ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/809bea68f41a7857ecea197ea334c47d7d5fbf77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/a1abe23fe36d96f2fd06b348d8c1a2c1f9c6372f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/da41453db77a3bb7b91373125e78a149af2789ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585916.29/warc/f41e4fdea6d4237b7f3929cf22c3363727ace4fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/046fe226d353e8b93ddd5945679c01fcffb5868d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/1e22f987c16f1f89538bc420a03da9da7030db77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/72fe21722f78962984fe26a0efdc54fbc341228e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/9f0eeac50f545e75ff5a42cfdc586e10ee5f4010.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/dc39467beb74b4a6f6811d9b61d56be1369279da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323585997.77/warc/df8b42737beec4b1922a9520f0800e38965da1e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/1e98ad3a56a32b41df17de0aced8e55ae05e0e67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/ab4a48d49e6f4728698983288c2b6dc572d9c6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/cb35d9a28090ce7b85c3147c8dc7623f0d7d5579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/e95eabe5ce72e790275982b079b46fdba2490dce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/ef246eed12bf9fc588da69f980a62a340246626f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323586043.75/warc/efb581aa303930dbd7ff0463d868dd3dbf4903d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/4456d1a8742cfb098e25c8dc7d2a52b52b776716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/6835d4b98b8c19cb1e82ebd58cab709beb25b303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/8d3d13c100b12117ea2cedd047728632c71681a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/8da5e3e3bb58c128d6b0f618d3c3400e77fbd81c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/ac1b4b78a7ca29b48376610298f01c5305bc2a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587593.0/warc/c45432b124d7472bc373fe688915e3838cbf3e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/145231101c8dd23248bc676dd6dd4c44ccc16bea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/159795c06445f4c0a28e0252498554057814128f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/24f821804b8df9336e304e9a4b8e8146fc78dab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/5e8eb5db58c71d221d7b636dba92ab3872af70da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/72830c2b18d8ed364a00afbaaffb700e10ef8c06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/be214cae489c57851ad917938efbf6cda8fd72b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587606.8/warc/e84c13dc88416d384f407b537959a63b122c5c42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/28d65dd2675dc16cf2b55fc56510de8c1b8fc171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/295d065818a94343533443204fffe7852584f310.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/3ed6fb0650b78a842143a832cba61a3239ad2d09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/56451d498e7a13ddd603b163fbe404f8f3c80d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/6bff2838741bba03759e9d657e8b6ec4da5197bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/7bf9afe46f32f58a5259a81cf4f221ce4b6d0057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587608.86/warc/96b05cf37cdea6e4836b027e8ccca4dfe42b1349.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/103ce0e712081bc5ea7d9f31181f6cbb76f64498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/3f3f4e60773612732fb7e9ec8bc2400baccc6330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/aaf92b5c6e14a59f1749808a0d4a819dff5f31f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/b0b2b4c22e362b32e31eac6444f71800f3f60515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/c8e809ea1c532f8dacd70610d632214cc0f3e62b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587623.1/warc/db1c20367bdb2580b007488b246df1db6802fb04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/21daf06ca22067590d9a807e9d07c08be8456855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/269b2bbd7c5d5cbb68e7d9f61727b84375439b2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/38208107e10f1605a6891adb049aaf42a7191d37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/48523b88ee0a7816167e663ec0a3a09839706c7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/ca81fb4fabab8694f895e689210152d589be43de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587655.10/warc/dd0b54bfe87d5bf4d35bb0adf64dda7012d47181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/32053a74031e227ecd00dfd7ffb42e9c173cf964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/425f943b9cae64289399f85d6a84d69783b8031a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/577bc6cff40d008a05cd27e348836b0245ae7548.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/96a74916539f49f24949fa3ee3101bf65fa3a92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/a6d910bb9ad5eabd663bce2160066066465fc74b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587659.72/warc/b4fcc5c5039c586f5528a681c07adef967b073ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/09166b529aa7b49f9a28266ec2a8ea6e764695e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/1217e846eb2312569101f47a6422d09662f862a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/1c265093f8298aa741242e2add135c613b187c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/24d4354e96585187d9bcfbfc195353c0324b24d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/8fa3ab827c0ba508e0459ebe89efbfd13bc4a7ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/a086ceafef21defcd4f32930646ef995accf7745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587711.69/warc/a3856ba1084dd7b24a9cf98656840208ce484459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/1608f1a96a62c254bc64bbdf972c6f2da6147477.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/19d7710decec8c63687ad219e3fd6f5bcf33437a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/63e3ff888b66389121eb3113dbdc4831dff2d210.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/7c5f00d0440e9d67f495970a5db3f12fd88710c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/921d7c972d5040d7c11be63c01f2d9e4cddfc493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/aa9699cfac0e260c9362f53b954b9b130ba9d047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587719.64/warc/d2d9f37c1216f8683a90e49d5dff9b7bd3d12a2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/467d3db4f9095117a680db27ae322ba90b2d8d0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/6b93f2825e620dfeee126af4061b08b9dee92572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/ac402714c2de90339feec33710394fd29af1be01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/d6d76feeaf0d0587bd7060a0bb17cca132b37879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/f1fd41809e41be756accb1632b7ba9e22a76739a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587767.18/warc/f8c70aed9b2b4055af0655e534145e1a34b93050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/1d57e14c4d2d0673c8902c1c6117d6b60a50cdab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/527c25b1ee1713535ecb57418a497c04b9973c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/64ac016d20202a35ab5d9e0929e44d1ea55a1c60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/a33fc26a68ae0007c1f9aebedb602c97c41c634c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/a60d827af34ab94be7a95b67fadb1d73ce9954eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587770.37/warc/d71a5317a12c8a5f8660ab2b98a35c9137b3d01b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/1e71e61583c85708d0feb1718e09511c80cd6aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/3ecb1ffddbb3efbbd0b9b5fe62f1a3336ad6d8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/94700003dbf7c72453f7901cea134157a7f50538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/a0eb57cb813d2ac522d0418ba0c5c4af66ff5d6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/b8a0903efea740b0f54ac92f9640fafb88f34c65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587794.19/warc/bcbdb403cc0d3401918ba9d11e8a395da53bf27d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/05526027cf63d3a48e40f9065a743e7360dda2c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/07bf0d84848d2aced056736375f145c4184e0fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/4a471d313040ebdcb3443cb1d9e31f84dd239ab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/4b31a978ea17843fe108202c6e0975a06ea2b062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/4b353048a2c1be7bfa236c7edf68aa2590282d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/91743da86f865201a040031d8ac57e706a528e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587799.46/warc/b556f3e698e7e24e1871c0be4c4be525807b7206.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/735837ff455f7b6aff8902c5a2822be58a6655fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/7e7d43a3aa208f9b9c12062a64c22be944442a57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/a5fd6d75c8e99223477cda82853ff05b2f627644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/a6b868bda40a89238ae5d9b735abe1d1a770a17d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/d5cfc20c8a28cea5f5996c1e3c72c2081094fa71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/eb27c28c75bf50bc39ad63242edf83864e47d464.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587854.13/warc/efe753a67b0a38c7cf4adf1cd44482dc9190c0ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/0bcc88340e282da7565e165e19c885571361c7f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/4c7f9f647b5bb03b68f53695ccde43fe34f3dc8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/5858f25c03f4af8164cda93e8011fa16e591a775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/873d15c5486678660127d4ffc2caddff7aa0dee2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/cb552ddfb339692ee7f5db298c06d554f08a9224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587877.85/warc/e9976282cd23bab192598fe1efaf9952514e4710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/73ab29ba76b1dc002478c3a1bcaa4deb973f87e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/7fabedc76fd42cba64ee6b7738d5bdaa313bdbbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/8aeec029f708de08480a9858e32b8ed6e31d4cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/a219b8aeaf51f45759d4b6fb54f69155f4d83f34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/e2d6635da39e4ad2c6bd187e23413dc7302b0a67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587908.20/warc/ed778082c97b430c450493e00f7092540c4cad19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/7483895b901b22dbb63f46bd9328dd05b76bb97c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/7b8fca72ba4c0883fb3953394faae108eaf2bf41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/94f098444e24ee5f27590d7ac64f6aec5f03dcab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/d1e1a9d1252712364f4a89a32535c24d0b972e0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/e0a9589703f0e575a04601f1fee360b8ca0c6137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587915.41/warc/ff4fa6ed2d964a1bc0081f34ab1075d31bc4aba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/1b1e2785a212e7af5fbd56fb22ef995b57cba16d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/2f70cd5aaf813737104cec0207129e89ec890c47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/368469a31825aa210c853540420e38b12bc994c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/5cfda94974ffd0250346d2d5a54cb14f02aefe25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/a1e681b931927b365aac6e913877c6e2156591dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/ae2c64d9cbf0758f07fa89b3152e1f10f26e79fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587926.9/warc/c5b0532a22f1b2d1e05f526c59177e6fb748e41e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/1adfd8fd077c8ecac7f67d8757bca27772c3b22d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/594fe5d08749d24920795f7714d6c4011ed9b9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/749a158b8f3493015fa2e5d538479976a747a8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/900968b630247d2d29b1fb65c1d780f1b9ed4b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/bf1a67866513c644d9b9cd7c385b52de6cedd4e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/edc94e6ee03658e2a6f19c6516eb4c046bc4354c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323587963.12/warc/f926acbcabff38b1c166909325b9fbac0c8dbbd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/2c6c355ad570e541a6812105c4cf5d2cf13f020b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/71591ae7b014eed4a6e5c4cee47a7c43c9b41c56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/7244b25cc02a9449abc5f107c3a58ee29a2524ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/d4165a390d596641d49dd0d4c6c03a1acac62391.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/e34bfb392f2879dbec72ca60e10e27bcd85f9f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588053.38/warc/e417d29dfe394b9d1eb3fb28ff5ca1a5c0ade2bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/4c87bc31e9bd0b143922a98582aa1619c670cb55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/6f08521737db482fa5aa0c480553ac0892b860e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/72227881a4b0c77d511ed2369fbd98547280aeef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/7ce9b9994d141c71a62ddf9c4468ffefb8983ac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/aeeb9625bb1401e776604fab1250b92a1be61a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588102.27/warc/d13dc3306f9916588202fef6513a6ee32f13a012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/4e5fbcb425d3ca1a4d201773256a1f6558e4f3a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/90142d06b76c8487f787fb907ed7f34210ed3525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/b5d4a0776c61dbb229c2daad4ec2397bdfa900f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/c4fad19e35610178cbf37ea0594c00e36906d749.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/ce5bd052719bff45927733dd30c4d755c449f0cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588113.25/warc/e84467719a41de757c2bd21bf435b1bf720d92ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/2173ddebd453f27c23e9e995aa4cb456dc1085f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/39754b9c49aad100512a914eb072e20ae074a550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/46de8d4749f7569fd2856d8bf4dffc201cffa96c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/89490c017456fbc193df366514eb2e87a076c443.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/9ea856213355ff5e6c24d64ddb428d1aac0c7353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/a4b697dfbb7a8679538d2aa27b28298b68d26342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588153.7/warc/e15bef275ba65b80c614722669ee3a48bb17d85f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/08c3be2e94db9e8ace7b92d1124d04c2594b2d6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/5cdc8591b19b87c09b972b45dfdcb0d5a8011cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/714843363cb3b66c44b846275add3e36c4f32979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/8e367cc25bed86e28aa171eaa0432ebad76251d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/a281d101dce468e7f03325d2d97b13037c77ecf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/ab24425aad3fcaa1a61ded3c629809e3a137b7eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588216.48/warc/e29569116206b792873e3417414a9389d21e6ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/077299c60ebcf9fc9eefc0255e6cfd310ed230a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/0cc16e389b7914e148165086c4c2af76a326987c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/5048d909d8d9fefb7878849ddfa86f0ea2a9d89c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/616a01b9bd9cef9232c2b6c12999aa6cf8fab544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/61dc02bbac90159647634d4a02bb5f1ce9979e39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588242.22/warc/b5cf00a1e46b8b6fe15107539c2338440ed06242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/42eb8ef219351552dd6cf8197c05b34e8a91bf72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/8aeae44330d66798361877bc232784621085cd44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/a95972dc20c3c8caa686db86c48737cb67a82613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/aead2c8d0dd88064717e4df0f59dc3b1db5a5d92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/afb654c840f81d0c806512051adc436af2bbe3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588244.55/warc/c17434e9a78d857b7db0b734a701f56697b8269f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/2ba785afc8a1bb739e33f4e94e4daac39dbc5e16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/2dab70fe890f6da4d15791aa2ebf2f70df3258a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/8cbaab6f1d5285cb9fa53c1647c14d657260ea1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/98882ea0e548838be33a07e6cfc9d8aafef14fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/b2baa5333702ab4adfccc7a38a70e16c7d81f637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588246.79/warc/b5e6f8a82fae0b3a30b1294288529149dbe5269a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/00e32ac931c7c8a2914bb11fe5cd2e6e59a42ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/22d97fd05b544017a5f007ebc6a6b3cdf165ee7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/6bf1d9debbd981b1d58024415c3d2aca5d95622e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/6e4b3db18dea4a9c50fe7c4e763292157eeced61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/76322113d856792908ff1a109ecfbe7d663bdfad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/afa4f6b00ccbd497b3f5bb00ca5ddfa57fc3f688.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588257.34/warc/b4d35e0655a4d83ef676aefd7f207f9bb97eb6c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/67f3f1064de68fcf7d6063ae71936a8a4fff4f2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/9cdcf430e6cd411c574e6afdc075c6bc04359869.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/9fb074d7a610b398e7babb6e4373883596115a03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/a47f4df172c56eac9b5f72edee31ba64887f5a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/d0f930f93b71e63a440c03b10ffd408f6b7046c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/d41b020cd3cd96c3508ae17ef4873f8c586dd4a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588282.80/warc/da7dd9d70c7923104d9ddb2536c0db779f3500f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/059281247dd2da738a435d9240035c0e4e540538.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/49a76b48e1dc602ce564580469823836d16bc7d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/beb4176787d73fc83c78cc216fedf487c2f01fc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/cd7342d8644f05b29fb06e06c992a963098cecc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/d45ce32e4485cd074aea676496fea8c89637997e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588284.71/warc/f222e006ee75d86573554be8d44290f3d7ceadc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/315e44c5378b4902a183da139c989e26b101d9d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/4402c48bebac3aa979d433ab4c359b8efcb76ac1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/78ef1f42397589bb12c0aa070f6d05868bf1ff67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/b7147cfbb66f86466ce3dfc05c5214f5035e10fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/c04d3c9ec023c6d568bb70db6fd3d509f66908ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588341.58/warc/c9184d5a2988d7078afb4775c371a828c20ca502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/737d47f30a863d8b491b542ceb6a926b8e7f3f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/896b969d624c1431c212538e80e4853a8fc15725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/a1c7d7b78ae4616dbdf5f789d4ff7ea7f4b89392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/d5aafcfced4e755bcee1d3cba8e9045fe8c4faaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/e60f6b351c0909df1d32412b40d68baa04d87709.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588398.42/warc/e8add3fc74320913a2445d984771bdd4ea3c8ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/0cb34b659cfa61336fbdd31e53f76d9d00e51ca7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/68b9c7fe5c035af16d3c5799dc7677f100b91ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/7537342caac69147f9ebb33a45dcba86d83d2810.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/81b806e9a8684cc4ab6d637d0a1aaf27ab33c521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/b0334d0abbce28adc5ae3d3a758bd2cd328893db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/d8d238388ae6826fad77edd2a26ffb5b45b3d054.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1634323588526.57/warc/eb0789e332f68192b3ca8bc5e1e6cf069dfce12a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1d03a231cbdaef37d8a5b4887c4017d85e9cdaeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1e89ce6d42990fff78c8977f30162829a1f619d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/1fd490604c070555a9a7086e94d98d4c41be357c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/244d86f9310c6a971993de7a1d327ea435fe21b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/2606567c2aecd800a5c06da9eda50c928a94862b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/278aafa0c5d2d5a8f850456e19d8260be75887c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/27cbf0035840196e10d0fdc80482931b7eeac2d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/2a341bd9d05a9c1b58571d6b2e04d55cc674ee33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/2d2d743cc2b336ae95b38b163c186a4a50a4bb98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/2e37bb487e18048a7b80c26615ee817e0dda750b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/2fbbe0815a8916af11da41d719cb134279323647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/300d3caa41a4bec2dec042dfd44488ae00c1ac8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/33d47ab0df023e3bc58ab1778784a4ce6bf1bcce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/37c2c1b9bfca00a67d9b61cffa5dfeee5c9bee8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3929ad2649975cabd06ec1cb99ed4dab5753d10c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3a8be0adf5f9d705ed8d838caf24c716457b5e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3bf0b765597d9f229d801b210fabb50728ca4af6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3cbe04b4c47145b9d650d063b364be4d964e9230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3d9512f5a7481030bbc1e3901fdee64e19171047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/3e45dfbfb8db790d63a8a707d2778c754a5c07ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/4078ecba2863cd391ecf3fb3a006faba7db08069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/461889869c521de8d021bd13a87b4bdd40053ee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/4c96d66c43c895ce0f2997a2502558b7fa917c3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/4eb67d5c2cf22f5a520fba559cc5eb5bd88e588c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/4fb4737d874a2d84d17a991fe70d95843c66cda6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/5b8e04c45467f49ccb13cbfb5bc9207331f854fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/5d4ba44cd1f4fb1f2227398bbbbed17740f38fd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/5e67317b9c5cf1b635b3cdf029abb8da8dd3fde7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/5e937756eb084582cc8a1c0bf84f6229507a0ac9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/664ec0929d8d81be47bdf9e81b02c4fbbd02052f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/68d8aefb95fa8d9d99446eae24ba01415d55f26d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/6b1511dbc9e8d4182b5eb7d24125dc992ac6ec70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/71759731cc56770953c44975c2cda54c7ed588fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7529d4e86dc632c56a511a8b9963d3dae611cbc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7c1333eac588ce9fc45fbeec99720856286fbcc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7cdba6a5e79ef3ccaa606dae4135330aaeb89bd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7d43f9369227252644266adb958ca55e14fc886d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7e1eccda51330dfaab4cc4619cf7b4a87ccc5146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/7e3a067f2e03dd7d2c49cb7053e38843051e27f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/821043eee28e1cdef7d9f63a11e72e684feda195.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/824bea931fbdafbfe3277e8626e1e005fc61bc64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/828e3ec9a412dbd830c3ffcd53d51fb05733494d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/83a30bac372758da481804d2c63be2b66276b2df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/8a0c98c59ed4995ebf9750b5ed2e3b90d8a4afab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/8d75713547bce72132df7937b13499c5df1e22d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/918b8726782bc472ef997b861e832ea849fa639c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/94671623787038e333382515366c3098c3aa7ee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/967303827efd940065c6535437570c7b8f22192c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/9bc79bbf0d875195820deb762356deae8b246b29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/9dfbbd928ffaf83abe0a5bbb99c01f8023766510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/a27bcf6eddab8621aeebb5bf1f03b5a79758c71c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/a478af6f7e44a34bf717c53091546eba1ce8f770.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/b2cf8ca985bfa57faa2cde91d6b784230496f2bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/be6a3bc5eb56ef76db485af5c9fea76a64fd3e42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/c366dcd6299414f067de07c723c0d0b47671093a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/c6688d41b1f841be0f2f9ed008b832d174d71e45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/c7915d98caa45d49fd5a51fd5e8060551373fdf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/c83660ef5fdeefca2363ec764194458b0115eb0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/d1a7384c2cafb6b89daa969a7f5746497d0fec60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/d49c707b555a03495c3efb57a601ddf36e846904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/d834ef9b76d20d7171e1b3a1524241ae75be4e32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/d9387df4038fb28265b0a49efebd99121ffb7fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/db7f1ef8b1c3614f4ced6e8da742deb6f9c8686a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/dd1f3576b776aa615359591cfd6288585623d8a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/e3f5a8eda5608d2b9dbe8a237338b54f717edb93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/e513560f65243b870d06ddfed2e58126dc5e7471.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/e9e204769aa305722a63ea0c3d37932bc38632d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/eb82f7254b8623f0e1c900af5829aa2eeccee60a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/ed19174c4defc8527283c42e087a030132c0b60e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/ee1d3049f82f65dcf26fb6d8ee0f32cf5dde306f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/ee291b430ea063d4bce568e4b9fc849867ed36f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/ef49ef76497baa33363b96fe5e794d392d4c3314.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/f233897669d962f68280380797a555235f46c99c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/fb5eb1808540b09a37255cb98542511e05f91cdc.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-43 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2021-43/mix/input" + output: "/tmp/cccc-CC-MAIN-2021-43/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2021-49.yaml b/configs/cccc/mixer/CC-MAIN-2021-49.yaml new file mode 100644 index 00000000..4a73214f --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2021-49.yaml @@ -0,0 +1,754 @@ + +streams: + - name: cccc-CC-MAIN-2021-49 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/013daa0e9c09c96aa01c6d981ec8263610cd015b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/0889faa4cdee10141689d23582a52b41e26b0966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/08c3e17634929f3795a26f0c85866266982b60d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/09955fe9fb8ed7e4b5e9d31a643926942811cf19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/0a192a050cafc5433714870b597511f64b079662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1142ffc83a6ff01f4e482c5d1dcf05be62d8d0c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/137f6153429106ea405b0b233dad0c420d152a4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/14d20652ab87f8da7cf889cd65275b21f810d390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/153aa2f80b5a049ae1234b45b0dfb8b6bd50d10e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/2f0b73a1bf1b2933eb056cde80f567263f197a4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/6faab736f9cf749e91b88df1e3ea81f92424dccf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/ba1c64193bc2d4cd217d2a7bc94068588a0221df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/c09b9d0dcf542d9b439e4065dd57a415bb56499a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/cb5ba5222f2a904aba26f0c7455d4d1ad7c9ab26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358074.14/warc/fd2c34494d8c573e68669f4031e0017f5c413572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358078.2/warc/1d34b4fe2938a2e1e12cd18ed2812fd75dd24287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358078.2/warc/1df1bc7329d307280aaac86d3c0949a45071a272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358078.2/warc/2e77e6466a25aa95387123b45ffa18da91a61d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358078.2/warc/7cb64e07c9d63b533955a029871d419589e6c553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358078.2/warc/951fa1d0e48129aa9de9305f169cf87f28c014f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/26d4887af3236d8ca9e07e578a5882e3d9c09e72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/86c56eb504805a41ba1f563685ad5ad6894e41f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/93baa164c308826359e6bf785dee0d12941c1aa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/93d8cb33e1c643b33a481a7ecf94d42ac45bbee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/eed9453cf1fbf9d500f717a97fc5dcc3136a4d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358118.13/warc/f3e35e10d15a1e480102f5299408e6221462bece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358153.33/warc/0164f2168357d33bfe8187fb356e6aa58bd02a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358153.33/warc/0a07b29333691b0d2db42d911a71da8070145b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358153.33/warc/2fbe4e0735d89040d93de3d8c17cf6560d956b49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358153.33/warc/63cbcce00f798fcdcbdd49fca4878e0e75c489df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358153.33/warc/b974be69f35f021152cccd9057717067e284a775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/0ffbdc033767a01999c18a2e303732116bd6df59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/1d89f9e462f2b17ff3b877c80397c768a36672f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/230173b87b460f1a8a9b838d9ab546d30b1f083a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/4ffa51db05ad3933a10066fec034c26b65b2bd45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/56f47020f93bf09f3ff53fd41ebfd01ad4d31fd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358180.42/warc/d458ae7029059314257b113f9626c6110079db0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/71bc51544b257b9add9668d10a31a970595b7dcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/7fc1d4c4ceac78977404c1b8d6fb733887d1683b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/e094e7d67eb20e4a0881b960202310828a701a4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/e74bd5f4e47f62d4bc830f1fdeea534801877d03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/e8b65d0a4acab270f3acd09a4b3d402645dacb58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358189.36/warc/f5bd3ba422f78a6bb61271b151728968a0bed87c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358208.31/warc/08deeb1d0882acb9c37a53856cf9facb924c8175.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358208.31/warc/1d1653bb4dbf7cf9b04409759e11642f662a87f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358208.31/warc/5f10c57fedcd2b84dffff2ae84b61c853e8bd5a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358208.31/warc/bb9fa9b2a7055aa4971e1b151f1270b276b3b2d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358208.31/warc/d7f08ddf960602e42589f505e960911f2b41f046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/4e16d14ab1d7c95088b5d3452608bd35214fea85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/94048e98233358057b21522e6a07f6ae0676a3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/9e7c625a32782ed88191c7c562e2ff45f83d5c27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/adb7a4f65c6894316dcd45c1e07c1ea3cb97e091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/afffe83f2dd3ac7f1b191c527a76d2584449dd9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358233.7/warc/c27a67ddf66cf8901fe33488acd3fc505b43b02b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358323.91/warc/4dd2eb619d050a298d1ba1fcda83200279e4c3bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358323.91/warc/809d11133e26ff5243f4371803e5966079ad817d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358323.91/warc/80acde6722202e323334c5095080b800e48f4098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358323.91/warc/87df75e87becc20af427c82b9f00bc652cbbd540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358323.91/warc/f67021cbd5e0c08f1ef347556b6df933d076e9b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/078d55ff60417f0da28d20475ec4652eb29fdd26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/3d20de8293e59189551fcde2314c12cd29260012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/8f87ebe3f884c65e782a53857c1972893c5009fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/b79e6141bcfe78e77aeb175a23b65c33a15d8a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/bd285adceed809e3913cb5b79715111e4047584d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358443.87/warc/c92e9307149bc2234ac3e43916cc1dc9548bb234.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/06ae7cc6905cef0731bc1dbc2de540e88b0e8585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/3b175712064c79449b8e2c426a323c9372f35d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/3bcf62af383fe3d70988172e4d06bea57cddc92a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/594146410bf2ca4acfae6a78ae3b26f53746530d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/b2853ddea2001db265e60fb501e0aa0ca35d6722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358469.34/warc/bbb954a03426d36b400887a95fab9a3106bb07a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358480.10/warc/43ddf2bb1be58383885f82923a8684db59aa0798.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358480.10/warc/46dc4108f80b5b806cfcb8b623fc44c36e34ca71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358480.10/warc/bf6cf4461b840f56428d3067cb43818f465ab122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358480.10/warc/c847323cd1dabead98d4374f096abe0ca732b282.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358480.10/warc/c955563df9aff93b1628e68ca8040e6066a681c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/195a6b4833ebae98bedf749c20fb0cc09aa68e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/484b5890ec131024ada89a1a1bd03a25af3ba860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/969c26989cf68bbc04646341365031f28a92c7c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/b48370611351550f55ef8217eece09eff0f63ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/ccfba2fa3556825ee37dad9bc76d66092feea349.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358520.50/warc/e974f52fbeb537cc806aa941d4945a9183db86e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358560.75/warc/1c020719ecb4ee33b0365e2ec8ec784ecae1ce94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358560.75/warc/27ba3cab20c16b964bf622abd7ad2f5a14ca7023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358560.75/warc/4d2e93520d44375a0d8cc6326385a4e5c285031c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358560.75/warc/75d666d359ae35676b0a2bf229ac86374d11bb5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358560.75/warc/9fd9bcc5dfe2da2dc07bff310b8438eb8f5846db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/47e012673d4a71a05f1b9ff630aaaf236162f747.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/8d1bd70e3605f26ab9660c2a152a90be86a01a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/9624cc6aa1926fc702243ebdd2029265be3b98d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/962dcf60f9f90a8dfceebb2c9a2ff0a645b4d75a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/cdfabfac469af26db3f18d3776644a94a9ea65cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358570.48/warc/d8a6e304fb4b9ec3f38f3b0329d6eb3838d33c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/273f2457b2fc030ab72ea3a56c97dd09ae842a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/768efd7282b75dab228ade7a2c7715a304bf8de9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/7abd866e9fc66437add96bbf5bb7033c3ccc403e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/851321c36f3aff552a9defc173b36ea68b0b966e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/9aef5284b0ad35bd081c2fa4bcf75a1433ad508b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358591.95/warc/a839bd814184f5879cb2f1873cec41f0d51ca00c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358673.74/warc/2865e792757977246708eaae9faa9e316b38d9ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358673.74/warc/572949264e248fda4404b4a59ecaa9b0a9e62079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358673.74/warc/649d86c9a21a73ffe84f64a30f8aa20efe866308.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358673.74/warc/c6820988f66073dc821a26895e18a45c663de322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358673.74/warc/e3ca7f294c80ae4cf960a7b45d24d2dc3fcbfc78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/63149733652d18db1d67fce63906efe93a60d7af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/7c12444821fca84fbc6bde0a448f7e4c6657a852.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/96005d3b66b2220028b71abdaeaedc09001d853f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/ba392433aa59dd383dc318fd3e52ba98688ef95c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/dee8570485c20b03f06bc8242986aff286f7dd31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358685.55/warc/eabf4c575d9b048566b6fddc1d0b718da63d6a87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358688.35/warc/23b5de818156412bb523317f645f6275d5f93aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358688.35/warc/61a4b898e0dca00786685249ffb9d399e17bb769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358688.35/warc/8b2be4521223b0d0c20e2b4994a5547c3337399b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358688.35/warc/9da31e1d914194db2299675d89b4426b01f103ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358688.35/warc/c2cfffde43eb5e9af7d283b4d20e0bc9109f5a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/10f4d2382f18e2bb7d17fd658e1abe3bf9d6adf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/3ec0359b3d0b9872dba2d54048c860330bcdc3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/73bbaba304401dcd3c23617a8c7a8f9829aa6c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/c388d2a171f530ba3acee3c8d0b4a1ebbafb7942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/d57beb1831ff00c7277948f5dc5e4689a4c84fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358702.43/warc/e77bbe51c42e081562d2ceee3623584e6cd62f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/5fce9f41c372ea61ee84b676aad92d5d0b596865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/9132aefd2708c5f75ddf470c500f74b8920dcb2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/9dc2493c5beb2f66cde3ef48b406a3afff151c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/a3eb4191a5e7e0a75092a847f96309b17b43cc36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/a50def8c23a668015ff34423cabaeb849f5c7fa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358705.61/warc/a5880c65e135a445162516bfa9b0834173b1bfc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358774.44/warc/0e251c334765efc6369e133329d2e08e143858eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358774.44/warc/aa3dd993ae4b8961066a01824ef852857e4c9ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358774.44/warc/c4a897e8c60f541b61002a344433e32a5a923fda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358774.44/warc/f2afc9af456e83a592c7a7ed26e2f77d92b9f370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358774.44/warc/fc83d1e9d0fc8cd65a1e8b15abc1f14c8ff6a4ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/0e69d22d798dc7a9bb310d2d165a3bfdcf532b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/0f27cf29ec16a6e9a8ad1bcc64f7a7d8584c004a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/1caa0bd1450c3c43f40ea3f993ea7ff2b7423972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/2d4fb534805765cffc466fb785dc5b6e3cd8c0cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/65fa2035ba1675e85513fab18b21630ff477e377.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358786.67/warc/c39028aacd4ed347e21916346ce56bc26eb1c4c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358842.4/warc/14fb8d8f00cb9790f5d12d82d1ed878a605e8070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358842.4/warc/4a896afc1490e3b65827273c6e4293f1a53c0031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358842.4/warc/751b79aab17ed89eaef958ed515801438f2b23a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358842.4/warc/cb8cb03ea4b7cbc7007885ce8fc76471112a9728.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358842.4/warc/f611068a395abd1bfac99ea0e72919638246e20d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/097aec118e38be7d3dba1e589aa9b5da34815300.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/21886f4fd9638e748f026c4c476eca5f21c895e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/388e0e9252f073910bdf75823d9ec7e6087eab49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/56f84998efd8c06597b9eefc1aad682dd065b609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/69b41130a8697e7ab28136d20d5e1a3fbb107847.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358847.80/warc/bb0b62051bf0290defb71ef4cd219735ab6703cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/2b65956c08e6184eddaae38e393dce52f63ba1cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/366d1febd932c4ec5c0e6f96a76bc0be6811595c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/b9134db1485a2fb97d6b6abf8d8b604334b369d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/e016a2a99fdc47d7c6da2e64e2c7cb5a2d79b120.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/e31361145604938379618bb16b7b1676be440cff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358903.73/warc/e87598084d09777a63b7ce5e308297eb30c9c560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358953.29/warc/1c05050e347972beec8af2d1185f9fb8dedd2214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358953.29/warc/859a37f406df5226b2eaa863b8987a56162c153a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358953.29/warc/986c3c4128e082fb0f2d82523c491ed35cd5280a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358953.29/warc/b27fbecbcff0d08b5861a42316ffc0cf99958521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358953.29/warc/d48f1fbbdce9b337354f4233487cde4c49dee872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/460d38008e848bb1ff75460cb5ac5b89cb24904a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/5ff21f56681dc2ecf94724543368a8ea28d47da8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/90c077bfc4b7c8025ad1d11162c6763f4032c0fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/a1ea182dd71b95a1108335776769c44e03c5a0ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/bd25184f408a714d3b3935547bd1a1e05aee0d68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358966.62/warc/cd84641dd1a8f9ead98b59bb119a41b0ed0dc6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358973.70/warc/16d381bfe3b8ccb6a8d5c6e5523f11413b668fae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358973.70/warc/3d4359425c193ecb972743f42c5f8ddac860f78e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358973.70/warc/6c8de340dd722a559dff78c84a1f1cf48ea8249d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358973.70/warc/bcad9d72ec45db68c7039ad48ceb3c54a3f2dabb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964358973.70/warc/d6629f01a56b5fb3fb64eba3475b35055743626d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/37f83e47d1c18de81f09e80c02e341b41d5887d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/88f6341b6b00e114a23c9ce949ffa986bb3a6075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/b6bcc526d28dc1e26ede6cd3778c5ac899fd6e6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/cf0eb9eaa678ef090452e2c26fc1bd448b71fcc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/d1a98fa06f6ec5173e6c266c36799e6aaa7da839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359037.96/warc/f571cdb1c22933151a736db276351146509224a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/11431406e47aad3c2e0943161ec2bbf224a37602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/2a0196d427f402b9001fc14ca1593dd274e08117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/755b8b8d7811638409998627fd943849ab72b1f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/d718251a89e1628ffa26c64025cf8484ce2c91c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/f7da050c60a93b99955e4ae29cfd5cf6477a61d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359065.88/warc/ff9968c9c791f1373c27f155b38023a769ec9acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359073.63/warc/5e9b372cf1d8884096c3143e0a54387bc1309c4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359073.63/warc/6276cbcbdc092939496e5bed9ef3601475ec9d17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359073.63/warc/c08e0c6cf0dc767dc7b2a4f8bdf150c490282f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359073.63/warc/cf22064c2b0241170087f9e48aade46d79eb18b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359073.63/warc/e79d58cebad4dc2f5818e8dd138aeff5da0f925c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/0eb3209a086b13866b5a7ef1539cf9d5bd8ff0b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/13f935454423df8c3185d6fa61e958ce8eadd4e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/290e2493a6f50867b5e9c9bb57c10de5d4e7a365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/6794a2e260ff832de4c54c2ccd55a8f707479e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/6b0c7db4f2ef7aeecdbe8b1ae9ea8abcb024c6ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.76/warc/cd13f5902d663a10c714e6c0f301968211f68513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.78/warc/5cf24defa24fc343d911db09d5820d4aa6944ae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.78/warc/5f64f0ce7386916adfa8d80aeea6f3d3c3d59b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.78/warc/87eafb875bd554c18aa8dc0d83e0c04a25ad0757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.78/warc/925c91febc9af8b5b6a446aeea8a8dab7f20be54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359082.78/warc/e71f8193f0e91c59da95ee160d0dec96b6812146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/1e65fbc36cc77d7944a1856f3b61c38208bb9ba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/c829bc7548ce3562d73748933aee15e7ff7a4a65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/cbdbe748d3462d85db9bbc5ddc241ea45b0042c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/e4098c8848df0a9fb60f111369248bf970b502c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/ea0f3594f8e31154db14786f198e0667ef93ffda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359093.97/warc/f3f54ad584d0fd7bcabf341045caa89d82e01148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/0d670428fd18e0443ba13583599153be6ae50cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/368f5bff949052583ba17b51381beef642058645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/551729813b5f033a0f7ddd9ed2010fd5d9a39116.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/b0580c2977f937818a5e558af82f11aad103d407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/c810cf1cb0614cbefa31a32aeb39f785eefd8c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964359976.94/warc/ee4964b3ee294ca531a4ecbcdfe3481410fa63ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.0/warc/094d890384393ec92d1b39bb2061cd501d8c19c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.0/warc/1d8e3ad83b2678c0940fb1cf1449d4fe443fd2ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.0/warc/64982f48d01c9a91bbec166a3a851e8e9400fdc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.0/warc/6a6da2763b48926b2dcfa75eecece135cd56c5fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.0/warc/bd4fd19576826cef9e2b87bec4bb2941320fa8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/185bcd51d033efe04230b450e462cc3e3c7f97e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/7bcd719fb5f9814af9a8f035af7418274e831cd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/89f51d0ef3cda54dad38e2e66e7742c417416adc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/a7701102e7b45f5cfa3e21703e47f675cab6c1ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/c95d6b3c65611e0d40864b413912598fc518e202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360803.6/warc/d3993acb53b8eb0f6cec457a6b3cbc89a45440ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360881.12/warc/51c62d67f3b079a733b1eb21e0e285e36c92ff36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360881.12/warc/5b6e03e5526cb5d3343f803aa8b10ee161998cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360881.12/warc/7cb6b4f67c7ca2504af3c2c4830e149e77d7356f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360881.12/warc/811566875c8513ac2e3431837ece24a5b7e15313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360881.12/warc/bb6462f52c8a7e32d6197772654cb478846806c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/318b6801b44098c548c992c5a3b581b05a45401d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/461a83c6f46cd9062e362bfdf0660d2a3fea54ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/49b4503c43bffebe1c94796f2ef6422dea928f62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/83a4609e2652132fb5466d1cc1273f8dcf10bd53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/8c6de76801f39a53f18691e3998c7932d1f362f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964360951.9/warc/e75964bba962a699fdd0f3ecc064387f39693ec0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/0f815cc4fdb7dd43bf6355f0fbe11a071a52ab7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/43585a3ca0ee6d64b72b87c3b71b2352cf89f7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/9f1e13550574e0f6505201fd92d407559e9d8cda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/a0338b224c12a15d2173c86fce8c6168548e0f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/bd3940a92be256ac060d3afb3a774a320153f392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.58/warc/c57960abfd693e8bf6693045175d0dfaab06b9ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.69/warc/3aab2c2d758f17b37f9c17aa256e86dd6828b3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.69/warc/473d73c36d4711622380c35d4694f5fde9fcfbce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.69/warc/4c692d711da95037f319c022b8706d8a36c8ec50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.69/warc/d1a4759bfc1a1caeaf449be53b4d4c5ff8aaf815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361064.69/warc/ee1e14c106dc1f6fe296cf4c3b47619c3ebbcd9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/1c9d496514cf07012fcf4d0d61c1aa2a695adf32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/2098af34a9864a85691827a163f8dba4f3a7d3e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/2ffb7a118c93d90081e28219320df05603b978d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/4ff0697a78f0f88300e06ca107a84550a945b62e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/9a3a232da6252c50d31738e5872cf518b32f23a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361169.72/warc/bf30ca0f6b9752be74de666729ea3b4dffbadfe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361253.38/warc/41958c46b421f3536737b0e91f209263a0531d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361253.38/warc/5c7fee88ad6b04ec1adb2b538512f481e3e3429e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361253.38/warc/91f2a000a4b910409c1374d03d5c23403ab34987.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361253.38/warc/9cf450fa1d05c2f478d2a262445164c58567d05b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964361253.38/warc/a0ae4f143d6b28a6ff32770d6b7b095425a07867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/02e1cc2e2ed658f61f483d05975a94a17a3758ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/3475c7296de69e5f2a556d9b85b1e23a201c1bec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/384764d65e1a389ad0536cc455dd14d7909c385b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/48fe8fb1cdaca998625cd5e1c5da7f6e336c01fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/d630ed49b023364f4b598b7d73e4593f9579863f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362219.5/warc/d6e2945174fb2822669bc5c8097c84808d56f81c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/0e7b20ae5c81ddb24f585a99818a531954d41808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/5daf54e326388f22db07f2325a00a6027875312a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/ac1a069dfca5d520dcd98f8633f877dc03035454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/b21d2dc72e9c970adc650c4af6fafe3d8f378de7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/bd13ba040e5b322e82875fe04619745a368eb1d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362230.18/warc/bf763a335d0d9e5eb32f91d82662bef78765212c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362287.26/warc/44844807df8ba443b1138ba6eeebcc9af6da3158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362287.26/warc/babbc214d3cdbf5cacee74637db1c19f2e4c8fee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362287.26/warc/ddf26b43d97c613935a295683c40e68edd6b0246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362287.26/warc/df80f303b25de94e8667cfb0efd92432ccd2b77b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362287.26/warc/eceeba37e6993b95e981900ae66516e95e31f665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/3c57969740938e6c9dd1c321e24f3d52fdc33ae4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/4787d699dca9bec08a30ddea0fd4ed4f90497934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/710e733ae034443dbb7b5ca6e95cf4b4283b17ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/9910a7647a4252e056e00b092fb0f75d940c62fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/a7b25695fcd5edf695707a5819063a3d0a54dc16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362297.22/warc/c4a452e3fccd699e6b88ebc00b90f52aa3dd5cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362571.17/warc/081f4ec92b6dfd6d4c5263698ec41f6226c9b114.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362571.17/warc/43655e6ed7616d4249bf2adb0782c290dd4a1e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362571.17/warc/65d2d4ed1fac3bed3cd521e1a73ddd4e6416d80b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362571.17/warc/cd78fb15d7418788e4365f895825db541834a381.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362571.17/warc/f8d1ec6b4e11c683d7478f6ea8fedcf3a3ffe57f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/25be20cd14174979b2b498fec6b168a2780898af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/691def238d1d29c997deebbc24742943dcdeebb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/a8e88cac187314a66d182668c76efc6cb35f5626.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/ab23dcb52b7db51b01249aa1c8407bab655bde35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/b01570d59c581164b9f5a16817b667dac637e313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362589.37/warc/de78f1b7124498668c3893f69d68c51299bf138f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/52ce3999bbb941c9f92eb6b5688eb32933f77ee0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/6231e47c819b8c55e390ac166469f803fff7cbf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/84ca75c7c406ad0e15a4c39a04d126015368b9e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/92c04b988e80581f265ab4c32e086ef08e974d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/aebddc1953c3ff8f86a39b80408ba5a9dee1ab4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362605.52/warc/d961c72c0489c44a2bb65982882742320b18bfe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362619.23/warc/325f88089d0f4db9f8fcae089093f9493dd4e7d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362619.23/warc/534eccf7154749383f69a1e0696a41111576ab84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362619.23/warc/70d5856aa3815afd9035f03ae86ad60c284096f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362619.23/warc/8d1f622c3d70b002481df6967a5fe893de2aab98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362619.23/warc/e4e6189a26f81a9956ab22d3855899398c7e1cd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/1153ca4a7abab59887ee7fbb8c92298ff6df5a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/2a1262a829f0c42dd9fd39410a06dae30ed73e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/6d17e53c96a0c44785480795dff22aa2e4a45c9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/77ed85013288bc7d280136815b48db3486f8f44e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/fa2b1db6590aaf14b889799809f3d7bd3d50451c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362879.45/warc/fbe5706898529b618fb19e78fa541b58d3b49584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362891.54/warc/3b95350e562dcc239f1ee7be29585973970048f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362891.54/warc/4296db88620aaab8915e998531b9bbfc8a2a1924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362891.54/warc/9760f36fc89a1c13435bffc85a1e609f5f7b085c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362891.54/warc/9e5d0eeb74e151c345029968b08432d406196c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362891.54/warc/a94794d0bb7ffc6d53c2203e832d8f4a03f7b6b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/137001a15b88d16cc7f84c13ca43d0f33a6d466e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/3c4b386773c2ca04faf9e3fc17c3926a7ca827b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/4c253cc57309a720d4efda0fb8dc3bae591c2547.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/57ab7743b5292cdb2df0cbcdc9d3fa8fc301747e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/5ef54cce18f4890d664effe798523eef1872d253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362918.89/warc/e2da8f3843749b915db360f99bc4b4e846deecfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/3cb6d242be38eb570abd703c775c9e74f38669d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/3f46056ea2d5d3696e8dd662e3ee9fc2192615dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/84e3a190afcad5b421847f4c42b23fc73876349e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/8782048d60daba06c1ce776d36c75fb6ed968368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/dc4740f6b36d76093a2c760c2126a6a6aee63c7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362919.65/warc/f884073d40c73aeeebe2ae16afcaba8375511620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362923.11/warc/1840bdeaa8b2e26d680fb88d96b222b005f6cee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362923.11/warc/19198d542ba3c5beb0bfe6c402a06279f9251585.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362923.11/warc/6974a1308e33f415c5dbadf939d9a0653df77f1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362923.11/warc/70b4c6e456f08e1469fcd6c767155ea8ee1eb500.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362923.11/warc/834311aa70a816f433c5e45d841053e819955e5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/2c28a3fd675e11d7bd48f32f190ad8d9cf3eb662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/6bf0ed291483cb4a2cad7d566b0589745df9c2f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/9758e390928456a967d51fda401c59e0b4967dc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/c03a06ead5be81db22562ba680bda6613e7f07a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/c1a180325da571e4054bf6e185847b3d926c71b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362930.53/warc/ff695f69fe014811934d6f6c7c34d8713794779b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362952.24/warc/300514555ecdbfd9e1d7e94366d111e586b62ed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362952.24/warc/7fe9e34433052518b85c348adcd30da53a7e6b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362952.24/warc/80a758b4bd64b83ec2acfb8e7f8f1eb9c92acbf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362952.24/warc/8b81d71e68d048134d3004e0c816e6643ce9a159.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362952.24/warc/be8bc63e365ddbf450bdbc6bcc73710563b276c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/1279d3b9684146cdebcdf112a9b9aa2acb1f89d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/1300f400e586dd68859ca633184d3942c72a66f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/302f0337cc7ac1aedb40f9e773dc4d54978e571b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/3958d9318940771722cf04332091a68a7d8dc225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/84f543fcabb67702b699ed1dc6684d656dee8549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362969.51/warc/c238fe287e757f80a1d87463dd4dd1c2ea725aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/172cc3748d21a18b9557155ba3496224a3d27186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/24139bef969ced2da2ce782aadc554d2adb1dc01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/250411745008e87ed6b4166d619f69288b062c47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/91c2983105280963be6c05732830d29ba7a6b639.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/d6da9a1c450405c6c79cfb1adc8432434ab72056.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362992.98/warc/f2445967af7a21f43bf0fa797f96cc5a3d5c68e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362999.66/warc/1baaadc4d71920f0bc6b6ebd06023f02a1d39011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362999.66/warc/7e438dda3edfad6cc8db21a31847aed10690f31d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362999.66/warc/82cd9749ae06858930bf5dcaa6653845d5c9bfde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362999.66/warc/a88e70bd6dcf27b374682213914c50f2626b2ec3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964362999.66/warc/dd4a2e4dc144e35c703c0bc30ca4041be2b9ee64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/46402b9545c325ef0cfbce5bec38241818bb3ced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/5f8d541b262a9dd767f611e36b3b67d91c04f951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/af6e1071fb4bd2a6f0d6c4a2960af4f986e81e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/c361534f35308d6c868a105b6e856479f0348df2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/d703fc182c1b665bf77e59e008a1ad8b510b55c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363006.60/warc/e374171a2fb4c6ca1365d95527f221aa03e82052.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363125.46/warc/0713752771e13fed0d3a71b59c6ec51587b204b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363125.46/warc/a156aafc0be85ffb9bb4cf745d87b95576c1ec87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363125.46/warc/ac751f86b91db422984bd1d6b6c7cefef9785714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363125.46/warc/c46ff5df76a8c8d6bc876e9edefafab80ec09088.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363125.46/warc/f3a0807e37c284a853d2ecf2903d1ce09d09d7f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/2a02b2c40447b07cd983f864ddb6ec4570312ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/4cae4ce59964ad23a492e4b7efdd6b7de6fd776d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/ae3a78594cd96999e5d6dfc1b19d7736a6dae5b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/eda8016e26cdab2faffadda6e37d186ca66bd0d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/f3776fa5fbb630714abb84f7827042382ea2b261.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363134.25/warc/f971cd3f4d10397bb9d813ceb45a0df92eb7871d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/35ee1c8446c852e79c42710b685c4bb9df9efdef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/6bd12a27f21b1d53135a1786b06118cbd10f9ef0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/6e19105c3e2c951801952d3635fd7d4757a14041.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/89936892ed7bb26664b43caa23a0836a2f73dc1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/b90a03bb726de847d7f0dc08e2a983aeceff7c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363135.71/warc/f1f034d83a5c6d3c53401cf5341320e68d27a8cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363149.85/warc/04b89a452b0bb870f385a9fdac245ce536125361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363149.85/warc/1927533f48a5a8c89af1184ec18339c7b71e01e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363149.85/warc/63f6074d792a04192e8d05a2cd254ec20e196e53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363149.85/warc/b43e60de903ea3c20fc37cf4c23504c2dca78751.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363149.85/warc/f702aceee8af8692f2dd00f99603bf47e285c05c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/3ac8c23f789eba4ba8a04d152c7f120811b380ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/6166c148c8867d5067c2e8cb290e5726de5c9311.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/857edcc72c634d281d5397133e0da80633ed15bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/86a71c02290bbd90b560468fc9b75d31659a75f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/de756f0c09f358778c10b28f98fc47bce1f91619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363157.32/warc/fac8dcddc4228423fb2764c2f63f62b93942b39d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363189.92/warc/3eedd9aa30b387cd6aaec5b06d4611eb69dad7d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363189.92/warc/6cb248495efa506a50ad10807170ad8f4ccc12c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363189.92/warc/79d800336ee379520d0b66b9443543b20f630f17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363189.92/warc/7a303cb98651140cecc721672787299d7af33c40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363189.92/warc/8f7eb9a6a55acd6a356219fdcf126a7ad1997553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/27114c9b26737b7602f4418f8ea7a9b582df6a7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/5492e929067c02a320d78db959f3031511b1158c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/6a65ef8b27b2ab188173cfc153eb5edc88dacce6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/81a7b917504e13ff69848dc2aad57106de819e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/b6c304c8e771a2259c69135ddc55d540f7cf2b9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363215.8/warc/d28da39297bd4461430a9ad84741a3ae0cb39a80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/078c5fe4b0e8ebfb5c5da2297329b16fd43834cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/1bdd729da73867bb65581bdc093753988ebb4cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/3af60a1d048ea1d631db7cc0aadf39bb0662d97a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/aedd3d2c0ab91e43c536dc7eb9e3dceccbeb7e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/d42fe4f5e87438c0ea5bb4f6b8b589a0c6e13513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363216.90/warc/d460b22e27d67bf14d1e8790807c7f675d1935dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363226.68/warc/06cfeac3edea862e5268873e90641f13e52124cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363226.68/warc/419e4544a793c169fb77c3604b0b856ece5dc533.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363226.68/warc/7d90bbd9ee32a1e4ba971153435fb20f0d1c355c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363226.68/warc/d7553b5e7086fbc28725d334926469a7f0396c9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363226.68/warc/e15ebb6ae76c06e098ca8c13b2b1905d0dad0a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/2f9214b7e4cd243373c034ee103a174ba668488d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/356ab4f2035d526ef88de48ddc510185c370d60e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/4a52f343a3234d86ecc58d3692e93a1f3eb71b16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/631b90a8f66fcacd71af0bd5b54afea741037525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/cdb795bd832a43044fb8d64073c995ad7097cea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363229.84/warc/ec433753ea69ea868f69812bcf001541a16e20d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.39/warc/0670840c24631484183dab04860d7313d96166b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.39/warc/0bc0b25b039bcd56a4dce94752ebb93fbd511a67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.39/warc/0c455606e84c2c0d29fe211275e2bd14652d08db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.39/warc/14a9f0f78273830450bb930c00702cb88c8e65ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.39/warc/15081e3c5506e5884eeb93ec954b2f1c61cc6596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/057b9717682f265deb493cd31c474a80c1fad20e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/263b4499f0c4d57fef6570412a8c58a78165f3c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/32f5de5422446e85dfb87ce68391fb241de689e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/5762fb1f696679e97374e7f99c0d421e30a944a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/6653bb974a636a7b05dcd318040ee0b70a9545e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363290.59/warc/a7e68807411c08047bcb942015cd5500db6093f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/2eee4ac8cc0a87796d7745b577d39e9d3b5b75c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/3e0230f393117bdc2c2fc14da1c9edbfb975068c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/61249dbc017954498ef0a703e85b1d840302fd83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/6888ef7d834b4fd653ebcb227d93003d435ff8c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/6aacf90b1e530c06ec322a5c115c075c70a6a7af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363292.82/warc/fa0417f9b0878751f2ca24aa4c54927f992b28f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363301.3/warc/209d88617b6e47c375d0c968f786dddc9f4949d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363301.3/warc/68a24e41a9cbaa11ffdc5abc23ef31d4bf538491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363301.3/warc/7a8437dcb8c3eb2444ca5203888cbf85cc0905ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363301.3/warc/a65f25890673b07e79252f40c5586995b92e1db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363301.3/warc/fef792c9b23cc735b8f7dd21dd6c1f1e62c8640e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/29146dd34e357b3652743dcafdb9e044e1209cb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/59bb915c0b115e356c5217aaad2f54716d3efb2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/9faf07967a369d906382f79f2eb32c2fd832c87e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/be3973c4382decbb62da5075c753dea1cb9509e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/d2604686d6c3b19b98cff606c5442d9d06f26159.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363309.86/warc/d8375c5a58ebddee1f0112cec12b9e8e78cb2355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363312.79/warc/9d553170d430a60b6939c36050903ade536dd20c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363312.79/warc/a2d1d61bc27484165114e2dff62c27c5b8bcadf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363312.79/warc/cee846053204f8b1aeadbc50d87a232906f8ee01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363312.79/warc/e0c0ec788b3dd691f829da4c5a1917cba1113067.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363312.79/warc/e345dd3543ceee7453bde5a80678dd4ebffb042b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/50db66957119ab5cf84f4a2405991903c83e77bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/633e3024f16288d7aacb5b9b873d504cd02d32e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/999f26fc260d7f17f5efa8264beed3640a7af4fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/c10971404100e7f02a980e2416c5ca78ab665e53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/cd1702daf8bcbbba42a0a631c81ca93b1ce3bdef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363327.64/warc/f99a80c61f40b10fdfbac602cb094ae490158af4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/0d008e6ccd5194856ad3a45d70551c68133d6d7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/0e6193715fc70ae9932324039897159a717cfcff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/b94b1c13cf5d27811c9d3cd438c15d0240e2036f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/c6c4c3eb6ca47e3836d2e6e7a8c7649dd2c3620c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/d2c6338acad74f8ff5fe01096d5a78af32d2bb36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363332.1/warc/eff81fca201496ec2c0c00f7eb81e73215d1c90f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363336.93/warc/5798c6d46edf25e90b1bed96c3f8dc85dbf47202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363336.93/warc/9cb85b21c2a1e36b7982ab796bae2b13c8681945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363336.93/warc/b2bed539311edc7b04cbd65de2d0914dc4a62bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363336.93/warc/bdaea854be215bebf0ba8cd6e883205c49501f68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363336.93/warc/c0f6488487d3bd52584386d58b8142b91553d049.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/4e0e085028fc0551c80c99fc62b7a7460163e339.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/76c3fd93852b7cbc982b2c669fe7705f13c89fb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/779926ffe3eea1653106a1c97dbaed25ef3fd864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/89925569d21546dbf0b0f67093e8c99b792794b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/9debc4db40592e7a6479b92388305295ee060c5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363337.27/warc/a3ce2a9e5cf2bf60e75a5304bdac7bd24e78852d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363376.49/warc/0622e8aee1723c6544d8f06a394d92a6c734b062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363376.49/warc/2e22b16b6135d8a6c0f8aa22056c7f18d95d200f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363376.49/warc/2f427d0c937b3cf58c081b5016f0063cfa80df34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363376.49/warc/c3bbe920ff93536411367286ec4a27acb53d2d06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363376.49/warc/e0eb5bb83777aa1116c013278aa31ad834b29fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/116fa340be9cad00d84a855878a7c5651d34a236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/413c94f9563e9a8e6acc3424d88bcb2d6a303d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/52c16b02a972fe1e5074449ca40fba02c4957f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/8a437c59875f0a50641d76b326fa8088a3d77587.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/9380c7cf4da3c1deb38db5e2cb52f1572fb4590b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363400.19/warc/940d36c1386d149000f44d4bfb8b5de546432e78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/14a53d69c48355d2eac21c162796a1e691448877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/2ed2783ef0efcf5ae7358a116fac0351a551fdca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/46554b0ac8e1bfca4345a01ddef0de95c8e684eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/93767ac225b98f0db2dba56b6f00a8be093ef52d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/988cedd2da393422b83a0776bbb0fb03f4681b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363405.77/warc/be0e4b948a16bfd3ae10ce4a8936c70e361fc80b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363418.83/warc/470b421bb7542df5e1a75881c802f544259b3c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363418.83/warc/8020b0a05a75e743b1d4340e6673eb8f4c867073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363418.83/warc/ae7ad24f3105d9fb3826cf7c85009b22791f3fda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363418.83/warc/be0c4e2da5880d403c48ef905818f7ac6aae1a82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363418.83/warc/e39abfe7945f8870cfc8035c7b449afbbc27015c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/264f7f8ae4cb452f2d440a2079713a41febe1af8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/5b67f17b61666b8e8b9f50193c3af9b96e963c19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/7ae239f26e9174b09d8e93cea698e760e1b35623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/c4ecc37d52f6a0a2bff17ca31a79943852c7fd8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/e0a43401b2bb64c623d909debc56cc95828b494c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363420.81/warc/eb398e14471adfb9e148ad14385df0a9d5d88bc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363437.15/warc/1377e34628f4a036cbccabff02c588706dd330dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363437.15/warc/1bc314b2592ce95db7864c7a5e70a95dc81ac850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363437.15/warc/45026e6273ec8921776d74c2969620de45d7200c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363437.15/warc/9c6b2e29371bab39bf6923fa6ef7841b4c33bb3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363437.15/warc/a0df406fe55bca6324fd32f3830bb4f25f1b04a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/08d820a07406fdaa52f56d870195a2fd85a965e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/10294a2c848aacda94f66bf7a95e0095d0502a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/14ebc6ed680fe2bc47aab4aec814a29e8d4b87f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/43863baa17617d0706b58d0bb74f47d5f310bb56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/b9e439c7af7f0b6e64ea0fc0250067c56be43ea5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363445.41/warc/c4015103734c33839b6961128bb5472802f59ce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/595d177b2ad9e924c22a855b19a28a0222aa7767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/74da6c614500d0c19f5e560d56ddd6760c20fb99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/896283171d0c030e661f4f00f8d3c022e4af1692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/9fa1455ebcb38d2f5cf19e7371430e2d1e6196c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/a42b0713156300d706423664131f3c787fe9ae33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363465.47/warc/c3425deec965b39d55f750f13c8ab8be20676a3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363510.40/warc/09eb1ec708257fdc17a8a0b901922e346e910e6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363510.40/warc/21bdf1b36dfc9c9b35011853af158061dda78bf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363510.40/warc/2ba5efa75a3ebec0e8e85c7f86064669dc4d5509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363510.40/warc/964b6e8cb01e8314aaa57ab06f596439a5c7f83c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363510.40/warc/f91104e0ec4a85c8db5bd3e45aa4cfe37e1ec220.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/09b0597ef648a989d743bd8c0e159f9522ace8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/0e0544e595cf323dea060cc9bc5d75bc6fdfc790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/35bbe66efd45309f1215729a36ae533773a6bdb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/61e39c1b150abc6544f86148d3915534b208cc64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/d5abaa4250cb01e4ef8bd79c425f0d3d29d7ff85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363515.28/warc/e298ad24818d49e220c80f3f95deb4a89c62b479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363520.30/warc/6206f000c8d20e69c7d6f2c339398cc17554e12c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363520.30/warc/7109ca8bbe2d09e9eaccdeb0508f0ec78964bc7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363520.30/warc/a43ae006cbe744e5962e556cf47985fb8b72981c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363520.30/warc/ae5e6fd419369684e6bf25fa15299a350d97c4a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363520.30/warc/d6700a250131a6268aea17e2761ac75ea40e4eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/1ce102706a514d4eaf9a70ed661138c7132992a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/3531380f9ce9bccc5ec90cfe0b753949d9761fef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/4171c0b0146dde49b36eb3c967a6a0099e85eedf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/5ed7dd02ca3f7a80f964d201ebc468d7fd6afdbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/64f04d9f2a18656272f5d7fd9450380fa3226740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363598.57/warc/8b8794483a3c8ed5b18455eb90768c344696dfb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/05669eec0b4c4e9aaf0eb5301b168d50b0d3348c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/287de23a3a6757b327873ba08913cb2f217b36b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/4e0e666390a4fe5297665c0535ab6e63c9ff0008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/9fce26de3c0e45de224c1a96d49fa119e20735c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/fb806f628525595ddd28a6f1a986b82865d62593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363641.20/warc/fb817244fb96cb337a8721bed1b14e783c1b9b74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363659.21/warc/13ae80b5543d82b81cd6123c75fb10f302db0cd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363659.21/warc/c730882d9a9d9ae7a5a960cdef123b5205828dae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363659.21/warc/d290f901f4f7bcb7d95779dcfc328abcb3ddf7ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363659.21/warc/df2d5fb68a81cfd2aa4b587c789fcf8495c24c57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363659.21/warc/ff7ab91a6a729490ec4a03210b1fe10d8ddd30c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/17522577d7e9fadc3739c20195636421742cab97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/2e8f0dcf5e6171459261217453aa9b877f7a43f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/619293327bd580ce5e4fae2730fce37dfc7efdc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/7e2d86c3c087d989c312f8f758e8b07368765b28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/913233664988f7145c5e44588e0b7a724cb9ac58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363689.56/warc/a793f937e12f2119b1c010534d58f3a8ad94c423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363791.16/warc/261f829dcaeb7f1a67124c5cfc43d30c19e0b548.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363791.16/warc/6c389ad250366c910a788573a8a77026945cf04b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363791.16/warc/96de89b6ccfc9a9548fba2e0b913d5e305dba2f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363791.16/warc/ae8a82666fb3605bb5d91dc338e75329445f7a48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964363791.16/warc/e97b61e2c1b2ba344023830c03fc95e445538f55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/0436d3b0c35ba71ea64d921e47b07f67815a5880.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/1e98c834daaf281f4f701302f1887d8f7b8356ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/5689345bafcbc841ef9ee3a10a171e1c4fbb56b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/bde46f995f362ec9e5b0cbb590d54adb1153b816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/cb644b9d8e9a5613f242888cdd80f65492f54c41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1637964364169.99/warc/fff634abdc0ada3f62fee7760af3eb0235c8cafb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/1651df17dd56021efec87ef7f3127bb1f01a2be5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/19a1f83cb8a10f7da5d9d1d2806530895f8abce0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/23b4e2220ea82ecb4d1ed2842b312603fbc212ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/2a1ae962c47a7e14bd9d5bd39605b3e150400a74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/2f33ff3a02269ded1f858bf9b241a2a240cee790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/311140877973f1d67ad61201db98158565364b17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/33f0f08dd184da76402d2b1fc223b48bff67804a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/3425e892ddb4c766ea9df19ce823f9fde5f90c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/376978d71976fb248d430293038f3d375a84041c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/3b30e330efeefd81975a4381a274d28d5af49973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/3ca8e1ca53a16d11fa5518cd12aaaeab3929d1a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/446c7746379eea7709202494f283dd79d7b20f1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/45487e787683e03449ffddb04743f8bcf2d836f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/486c707addeacd2d09ddf42513c757f4dc1ef3fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/4944bc248f1ba7fe4cc7fd92f816466c6c7df7f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/5be90052b347b2e7c0b83fe0d4fe3c2c45ece202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/5c4a947549968580451e6d3a510547ab6c7230f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/5e588e90f133cf1f8c6c6f8d115b69c6192397f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/631987c9acfebb51d92777de12ee7b4b088a4717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/6636644dc4a49f4da1b8647b1cff26e2c9fc203b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/67de22a89c577596022ee0d6ff75820b2fd55dbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/6ad3df20529337e722d453cdfccb3b1400e26426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/6b2b9e765441a2cb182ec7b8169282c8af0832d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/6b7032aa1a06b3b504acf9f1e63ad7c8348963b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/7035c0a6802a8e698f4f3334829018b56a5cf9bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/7197c5274eb446432345f8e421aea76f499b54ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/7943225d2bc314ef74cec7ac083901c3a88d187f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/7d494a2e119e1cc705bdc186f4e4e71cb6e73bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/7f00b44a72c9875adc4041a829ee9b1d9ed9d927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/83327e426f4c7317c4086a65835525a5257f3bd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/893815d0afc255567c137ee6c55f959cf68d4d92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/8bcb1b5995aec68768eb0538e83faa4f0929a0fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/8de769399e650e85137cefc3d8a1b82895df76ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/8e8f154efae8a8fd31a7ce6b96c089aeced517dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/8fbe74ffdb7acbab6bfd0acc2f243a9b698bbd55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/9078f556b15323ef5e36dbf56d04c5bb1589eeb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/91b21958d0e53082b969b915965565f819e11a08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/9aa9fea718624a5647ff520ea826c9ebd07ffc21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/9b7f241d893f1e2d17cde6dd4e89114a29b3c141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/9c46f947e9f5bd577d4d88cbe050e4063682aa8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/9ed05d1447b2dd6cacec9a598238494ca6cc6d63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/a5e2147a5589e342b4f27eb44fd2c265264eef10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/a82d93a06940d17e435c1e1155c2cfd473c5e475.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/b2c33434b389ed45037978409532dc45109ef810.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/b3bb64ef837fbab3f471f15c67fb070756dcc852.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/b3c2b06be0e594ad7e41dffe27d02249158800ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/b6af041169ba0542af96b72e5085b02f272e4e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/b7020ae1507e9c9b5aa0c726bbe7d9ffb49f60c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/bfab6150dbb666412ec76a66d7d64b4b4383ae74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/c04f9fdcdd334c8bb4544dfdd0788c8e79b2b3d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/c089dd87e80455447f4dba6453b0661b259c68ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/c327e671cd18b1c0e1fc62d9821569839a8ba3d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/c3fc884415c73803b238acfab82b24f8112c25ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/c96b3b547a38245704cbaf2139916791ec33da5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/d471570fc99008c863aef5c17325e76a121f1dad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/d519a89b2dff3511b7edfa0392a2e417b94515db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/d8195724b4cc428fed8316608f38a3f8aee8676a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/da1d2ff96159dc186da7d05289402a83fb68774f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/dc926d90b0f029d89caa0e816de4b6bd101f673a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/dd3f0b2bdf929254704dbde3185b82c9ba409778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/de09034e68f259a12c676ae7ae96d27d4be0e7bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/dfe9307dcf261a0c93a67149d294706977032b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/e5a3315d90ae24b49b3cf46e48a314b0c3f4501f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/e6a2b898a362a9f809953a25406f2e9c39b19cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/e7945381481b296feccf5a50d429ed45d8b9840c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/e934410e663e95e25626d3159e07781893231dad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/eb07d3dfb4d53c849bffa86606cde303bf6877a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/f18a05a0bcba0d969b64c290af93d763fd6bb005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/fac38ef1061c35818127435b1d0438fbb3635e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/fc1682a7c01caf53ed752401eed0f40c4ffa9bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/ff50586af4bec637db9f034a7bf11b149916f4de.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2021-49 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2021-49/mix/input" + output: "/tmp/cccc-CC-MAIN-2021-49/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2022-05.yaml b/configs/cccc/mixer/CC-MAIN-2022-05.yaml new file mode 100644 index 00000000..c33bbc82 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2022-05.yaml @@ -0,0 +1,834 @@ + +streams: + - name: cccc-CC-MAIN-2022-05 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/01957e4543d8e4f1de1cad5f38b18c20647ec4aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/0d0004189bcbfd5a1a13c5198534888b3fc437ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/0d214f3360cf023cf45c7d470330de5f63207904.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/152856782c4c6108188a75daae9f3e02cf7e6ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/04b5157e8caa4e096ce712d76c1092adb159f387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/04bcb6283703b292bf0a78f43269d371d87cb8cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/48bb52a3fe0c031052b50ce0ccb0eea3edbd3943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/61f4273e6b18e37df92627e835224dcdcf6e5323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/ac57ae53ac335d5426ce1f4f9a575be37fdc04d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/b0b44719a0d77bad407ebf411f319216748c3526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299852.23/warc/f4ffb14342497d2f64a53e48db14705fe0be1858.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/221ec79a24d049d9af48935e56a8c8aec08aa8ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/91bddf65e23d035c048b129570c2041b50750a74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/d90e4cd2a09ade032fa37aa73abbdf0fed8dc73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/e379301c3282765950dccaaae6ebed262bba9e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/ecb19008fd10ec0815fbae79de3cc3767710954b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299894.32/warc/ecd580cc1808639da1b34803e7a963334e26ae91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/1bdd05d007d8aa7a5ce8cb7b8b43e49e52d800e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/23b68216c8573e00a5a98c463b356021eadbaa3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/40a4a6fe5e09f59082e3d8be3927c77b9993b006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/51adc6e070b843367a4bb2f72e8a3189ae73c53c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/a864b36374627bf6ba5adaffb425a97b7856ab09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320299927.25/warc/d8a31bb783062c6ed18f14fe7c17b415d9ef58be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/05821c23d2ecf09f814e5688a26d2c836f245a3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/06481a74913dedd34bd44fe206de0f289e98ed67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/34d3ca6fc4327bd96536f4b43055500d08175f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/e09db6571c3c888407efe74c5500bef7a3bfc0cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/f158031706c636cfe682512d2cfe9b4c5200e344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300010.26/warc/fc17f68a439c2d6aaa5c64ede45a91948532e735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/011475a4640df63d3e233c1fa82e185bbd5d0984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/049f80e87fa26b1b0850c23ffd292bae28348cd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/0865a51fcfc5946ad8434d5beae112551f578fd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/2f826ff6019fc7150d610a7d4d27743962e8fa41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/8b994612347b4c7dc5caa2f624db48714e1ef5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/ac0272c1c0b26caab4f512bd1583e6238c185c55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300244.42/warc/f22d11a51520e9c048d91954a9641217bb5edcdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/03531c869147c33786c2aa1c6c7b47ff9023fa9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/3e0355aa328960f7bc43736ea1c388463472e2bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/432b823bcff8681164e36ac7eef6f50f19818186.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/505168545add857aa927e7e6a45c1143272cf58f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/7669cee79e1d1628db5cb22a54effda8f75fc6af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/a4b365fd32c3124234a0dadbd157fd6883b7a83d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300253.51/warc/ecde9824afcda83ad963ef6f81c1b0da9580ca8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/0c1aa25af08fefb47caf9e528fa732dfbba8dcd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/230cda806ce6ea2afd456f871e97e2a693e4c1ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/7331c96db6c889f81a70560194f2a53566f1cee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/7403a3acbd832a3ee22ea86b3e105a79267098c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/c153848757cc23e974d9097881844545b8876bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300289.37/warc/d01131674d43b0d462c31860273cf8bfba5c42cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/09d807dfe6cfab8beb24370cdd5cbccdebce0723.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/267e9b3025c477b8efddd392e4556e6b68f00414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/2880bdb254d4d00b84b5774eba3eac34f96f7325.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/6840d7f8d63fc5a4e93159a49ad560d34dd8a074.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/cdce8f2bd75291ec31a040f5f83c8cfe4b17d909.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300343.4/warc/e4563ff1acf6a43db7851fdf071ecec88759a284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/2fb5f847367e1af5c32cc5146d22a0a7d6f7661b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/609bf8dc6e6a3b0bbb875694d43d5a781d13ac06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/7bb8c64ae7d93f9db8ed49fc4da5c673b608f70c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/99bd40d0bbf3ccf50418bb92a22e8c7d63863b00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/bdb508900a525945e83deb16f982fea5bfbd82e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300533.72/warc/d85df9dff177c75bd9d33daf634de43a47fa5405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/1736e6c60a6eb7ef78697f26feadae3ee261b6d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/1d0d9882dbf43a3c58996ba4de975d2f755ff09c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/b525083e148913edba760c33321a7764fa38d401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/ba498cfd42670ca76e531e6ac245f915c4d0c20c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/cea72dcfb136a308092b1b7644c3432557696f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/d427ecb72523c1299ac27f4392e9f58e028b535a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300573.3/warc/f242270f8d926dd29eed4a0b48e9bbd1fdca103f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/0132626e670bce70acb3623793aa8bd3dc72cee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/6736058688d3b34706420b449cc2407588c4bb1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/942ff30309370df9089ce09220cda3ba6a7d3284.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/aa620fc805f1dedb76f4ea018f3eb3b2e96ee2d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/ae4312f9bd04b52ef0daf6b272044b37b7f93181.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/c4b502f258d2a018660a3a1a4cbbaabb4da87841.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300574.19/warc/d873b2145b9b3b23fd3d991fe585262225bc3594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/7b210de3c52c951c721e1b4917833c03a4426263.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/7d002d4eff3a6510167456dc75c83540370200fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/be016bc46b43c5fc600ec223285ff80f79e38f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/c5d6a8e9489549a076b7e540522e603e6d02bce9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/d610ca5d398e0f059f4878ac680a5413e724d6b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300616.11/warc/e72372765943d6b305e982a933dc0349da1fdf1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/34f255ad0ce2303a2264e1a4af2b9ea54a7ac853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/44204284766d77f54842f27932f87db1c2319845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/71d8db6d242599def70e425681c875627655446f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/c338e539638f9d0f3b1e6c4eab4505ab9f739543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/c75d59ca40754f4a9aba7f3a473c79bf184ba857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300624.10/warc/d76648d186c5b40e646b4b1776eda90146279733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/15977fce482be2e9720dba56f10a0c5e7e533bed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/1d449ac0947c029574816daf99ae0c64336eff62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/378ab5724533d568c8486fc43f3bbaf5043e32e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/4949abe96e3aed256a541c91e1efad0eb7ae9e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/7f6c9f88c4dca88931b8b79212751d13d783e7d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300658.84/warc/bc3d88fe4d55b186bf4e1517df8a51ed0b0faed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/3e752026dad6fe5f504e44183831eac39813264c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/6406eec1afef8751ff72e13bdfc4a47e5c36234f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/7e47db913de82fb4f765f3b835ed4d00d33b77e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/e8556dc0a39dcf2573a0b646768c0aec96c56636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/ea18d3898491f8803b06126d3e8d429185481ada.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/f0da4279c8220fd3d7df99f2d7a3f7596206a6ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300722.91/warc/f72f02143ab5a38de9ef13571d9bc7e2938dee3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/24bb410a89b3bd73d6d5271e0c7c2b9dfecb7598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/2ae58bf4e997d1e1a648aa423811c83e46130d87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/619d42c9a627be059850bd446178b719d9d0719c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/71ae48fe998d98de8cb43dff5a84b2b8efa1756b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/bf5ba81f757deb794b469904de904087fef8b6da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/c3e4dbb1efaf09d47a45c758e86ca534c7df199d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300805.79/warc/f268e5b48f5e268cc8f3e98ea96f68bf7a0dad15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/06042f0206d286e1e7db86641a57aab57a9cf258.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/1aa74cde34ac173b89fe6fb7d4dc01d4d4b95e69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/45908bc78e54d1d7c2449272c784c394048576cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/8796a87649e825534aaced86a16ddd0ed1ed2eae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/d3bfd13966b85eba6a4e688952ca05a909b89042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300810.66/warc/f8cd4205ba2f52590c445ae8133239c1fe293cfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/514798f5030f3417dcc0e6faf714d428b21bd676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/5a48497d150906216e7834ce2240395fac89c18b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/6e00f0c65b7c5c2c3285685eeccb23128f874010.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/72c6b9d84d973542ae364489e27f8a3a83fe49b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/cf87bc2c6532d70201284e591d6ee51d128d6156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300849.28/warc/edf6a0e859b7897d16acbb25ced6fcb95c64fc30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/067a728ae1ca13ce10da6407da8bfb4662dbea74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/11cc4f9fee7c718e03b52739c45b191b47bf4ce0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/4e9581a4fe1c053288bcfdad8b2d4ac43e062bc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/6ab6bf91b6d8833067a6899d2a82c70d38e4bfb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/e1e69e2804cddccb0e906cfce6193590da317c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300934.87/warc/e91b1e0fb764b733ace303a3bd25d0f5aa53d825.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/2632db44a7875fca7930b57870f82d4753d3ca89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/2757965452ab362b2dfb7b17c9c4d159f56d5f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/896dc89534fa4a350a5260798427a673a1bbc6cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/b86b214571e129d3f83aafce1693495df41ed674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/bee31efa96a1f1acdc6625ae2285e6cd69644fbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/e6ad6442f600a3bda2e5c042e2bb457f13a38b53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320300997.67/warc/ee44bbb44ebd2d0e2edfe31f1fd4b927ad989424.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/1487fb248ef751358e5c1a9aaeb2ef53b10bae4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/212c48d4e573624c28e98a9a396f4ebbe2b2841a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/2495e01e04ef5dbf06537db4a2ea0c9d580ab7fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/a8a24e231cfcece43c8d81696149082cc2a18d79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/b774cc9b89424b5110efab811bf687c1c56babe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/c5b3ab9eb77062d65f0ee11d5fbddf7bdf53ff7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301063.81/warc/f779a28f2118c78f7e510c6e8ceee107262ff227.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/3d119aca973e46d85192bf7d0b9ef2e26b6d214b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/92d3fadfa3b3c41469f1d444ad1069967a5f0e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/9727f245f38778cfcf9cdb1ffb795bc346f1e1f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/b2a73df0369029b366ee02850a36ff8e42d8d68b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/c2d0be009db854826efe819fa7e81a6f71808a3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301217.83/warc/c59a7e8ced0ec9eaea881c41ec6d3bc936416a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/4a23abf0e7659228056579ffacf26d359af7a82d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/592c8dfaa50dc70fa864dd1677e474d8dfd7daa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/764d882f7cb1fa7dae355277239690fa72ef0d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/a8b6c3b94c3ebeb3c6caf276e23094a574144cfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/da3022415d4939d0fc70e45bf8ee8c9372cfedb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301263.50/warc/e95f0fb63685912d6585b3ccf7b45fe89889f272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/0232b6f265ee86788d90e86d608d56c78a6c99d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/3ecf8f7cc1c5342744bd38ba7f63ea66a6b3989b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/4cb3bdb5a0ea475e708ed907f67fd2ca726155b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/70375590134e16c978258f535698c58088219979.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/a2ff91e0f51e5979d0983682d9275ba128d496f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301264.36/warc/c34a0f375b5b9a394ec48027dc744b7a55f1f5aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/1a20dbfec12e23b6f9b994223927a6b6bd7f0915.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/35c7cdf033500a18cdd4c4cdb6487fd7777a8ea1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/6f029ea6aa216860b1010c85a853d9a8d644de75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/8e3ed56c2f195c7484d68d39bf5ddf3b9e49434f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/a9d4a97f166f9e9ef3849480888fc299305c9720.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/aee085ccf97167e5cb8ddf9250bad4481d1d4781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301309.22/warc/e2f4020fea405bff40c8766be4f0f3861164cdb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/39a4e0c4d07e5a8a3d7657c5202dbb2563ce9a0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/3aace96c8bc75e47adc279824a17cec71a3fc82f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/475452cd72f5f08e7f03a06fc0c37ba8bf855b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/51fd55ce1365df73410af73eededee0babe58601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/67a80da524a702b4ac2288fe224d5e609191c48d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/a13ed082d59c434040899e7a32ea7aae8285294d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301341.12/warc/d13b82ed6bb82e9b273bdcfcd9058766af8a46e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/035644879682eb21eb0b4c7262cc451b6c6c75ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/2ea84116cbc5c63afd5ea6e36a0b338dc3de227f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/320bda23ba4cadee398ba135198be7fb8a2c408c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/60c1ce9b16b8c90d5548e5385813108b3bc16b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/97ed961237db1902ce43b292e89f3a200cf60d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301475.82/warc/f5409ff88ec68ecf589c5fb5459ded48b8d805a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/2b11e6619eebe8a97fcc35a77347565305485dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/4b08f698cf877fa25d44ba75adc5b20375c60dbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/4c6337add5d5f0609ad80ea73c4fb5331c7d19d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/9336bdcb0255803754f808e876e5b32ac11a1f05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/b22650e8f6e3af771aab6bfaf070b55a5aa921d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301488.71/warc/f140632999b9f0c3981fd4beefd555a2b94a5da9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/17d0800ca2115ec48f7745f71841c9573322d8d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/1927b4c24da31917c55c432af439a6b624e51208.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/2d5363a6026ee035729e7a02bb8566d280771fbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/426684e9532a22ebbf2893c1097df2dae8f22f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/595c3ff9fe40d8d0d7f2644e321d8e8d71707b88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301592.29/warc/dfe92ea280752773277ccff9b9eb4707cf2fc8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/103ce050d9c9f1c238d416f16c1be3a46b0d9b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/1f3b72bc40cf1d39af936dfbac9bb8bb9f427857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/5dc2a8f76ddc1fc9262196bdd56914314041a113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/6981206869cad4f95da729c149f7f1d2b8ec6cd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/a382cd24afcdb4707e46e79ebe4f5721189ce226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/d41f4ce35ac7d8296e6388b144a14731aa5077bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301670.75/warc/f1ccd92837a369c6a230cac3c69bc6b7872184a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/32937a620718dfe0b18746a9599a3f068ea979c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/5744146f3c08dee67e7d0db9dc15e071ded7baba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/590a92f24970fff67f21dfe2f4152fa404e390c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/6abb70d23955948f948168e6045475aed3fd444f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/b48200a52847cb1fa222e5f9867d5333febb0520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/bccf33312efb30150625be15d8468ecbdf26c6cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301720.45/warc/cdab0074c91d5ec738022acbf69b26a4d39f3c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/633cff19c94d1cf4cbabe726fbc21e7de273eef7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/9ecb3348911d0ad7bb4a32dbdcfdff883f58bc8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/a4ddc7c74c3c315acf87bfebd3b437362e8f1aa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/b7dd3dd4f3b613ed1570f260f26c73e1d4aba4ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/b91da235ee3853008e9c4a37a4309e840a4cb07e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301730.31/warc/debee804e159e1d5c049393447f7c93dd4b4caf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/0d7fde63d558296a47f1ff218f491def7a96e125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/1d83eb060d113fc47a563b2306a0a0e354152a1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/3f62009bb326d3f1b93608352daf608b37b4d625.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/47242c968fcf2ae0a42a92c031f29b7b08a26a76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/537614ffc23ac8ec4cc0085e67a51b0fa7c50902.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301737.47/warc/78dd0896cf490e4cd51b2da32f833bb85dfd0e38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/0f04b283ec1da803140c6ad4be96f09e0e33429b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/1a137f68d5224c91573ad4fcf9694eed1eabb6bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/7e17604be89ba508689decda0c097e1e42a6ebf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/b51ebef2afe30f934668a07052d15b702e8c6df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/b775bf9ee80abefc962b370027626217f94c600b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320301863.7/warc/dca335f1b6df9f588c9bc5cafe9aa2ca0ac356b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/25aa9177c0bc3d7e5bb83543b667c665cd8d184b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/320edd91edcc1a88a71f56db74f9a1f8a9415de7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/9ba66e22e05512b7264007541205e6a756a8e2c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/a43d599d4a8c8fe09f6b9977b3f115f34f843115.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/e1aab43c835dae1c83492da5a3c2e04bae0c1805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/ea9331745750ba8ea988a59b111073e6e3c2a10d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302355.97/warc/fdb3843f8eadbf16452e9ec79e901a21dfd695b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/550433f89538e1ce2ee5e4716fc59765927efd65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/69ae033081d3de450530ba9ebb9d5836130f8dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/9a971494211815bdfd1131aa27eec9c1fea2f87a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/cad2df66eaccf395f8cd69d62406fbe38f5b1283.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/cea44304530a155f5a8784cdae6bc082ee36b4bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/f116638b6faf8fd3f8bf397fb29551d5e8a47999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302622.39/warc/f3446106da42537a87c03ead0ab4e79bc47e125f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/47f9667e28a8d6a888165fd08809f362146e0829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/4e797356c3340dcffebca6a79df819099048dfc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/7318c2c4737d06f12f632063a44cc317ace9c7de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/7f8a5204892d3e9b0b1a98dadc4ae405c206e785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/890995405a833dad4e39d42ebbd6bb1533879682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302706.62/warc/d9aa4ece06eb96e1b618bcc8af12fe6ae4a3139a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/4be6ab76f106fb0df8a43c6b36d02a2e083cf9fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/73c40083aeb89ad07c5c543928d17b3bf7c50106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/a469282f111f7cb5f32fb36741d2ec544e7cd6d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/a99786af502243ddf04d0db4fec2b47023d1b990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/b68fdc6cfdbe6476510187e3c8a8a0a69877e409.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302715.38/warc/ee3ceed1235b2895c23353dc81dc49156a64575d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/2a1cd59a5777aab0e32990bc903ca24dc25760c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/6db2db9e778e3eb8c72d0eb60a3b89a949bf97b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/8036906d8ee5f1ff80df35b86a0ebd947425f3f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/9c5b315f9e1faf73cdb3b68601e6fd7257db139a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/d8d2968e76f7afbdf70d5274ba6105468e08828d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302723.60/warc/eda6471cf9e4e43beb9191acfee01de3333ee509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/0e3b28496dcbc0937b18dc86a14cf9384bfd7122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/0ec9a56fa7fa50d938b41fdf1ccb8a42e37e6b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/44871ac73bde2fa26aad6f34a404a036e14086e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/47a51838730b57c05dff0d1c9ef2707164968853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/a5cf91cb4a425772ea837c24bf4cca308a9ce4c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/d736c72a904db1685492dba06cd20e7fd4b5f52f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320302740.94/warc/f0538c48c880c0e3d1d89f316dac29c60d19e704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/25695e2630f859d65ecc583221ea8b29a175e286.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/453c0537f73a2d7fab2b11b6bb4d9fd2cbca7c24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/4616977a0ead43cf893d672f2baad6f5ecf3f5a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/61de1048a889afce77727857c5ed441ab98d0aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/a15b04ea36484cf6cdb51774f71a1b3c125605fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/c8df4236594d2b20b9eb231a23920e7b1e55fcf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303356.40/warc/f42b3bd22c9fe0af76c5776de65105748717eb10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/2baa124c07c5ba05efd82777c57d4246eaf95040.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/365da35f9456a4ebe065d177b6fff1e027bb4628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/6c1ffb9b04362c799339ed476813f348499851fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/aaee7237309fbfb51e2739168d38348c3923da35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/e90f072be255e07c66150e212ac5684a3894aa94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303385.49/warc/f717544ff49347f56331ca78d87dda91bcda904b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/08e78cacce994146b10a779bcf53455268313232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/09f038fb91b813c9af2d96e55b982cbf9b85c9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/65cd1244d36b932334519901bb069788ff70610f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/a0325cd4bb1f35fb141c3eb89ec3ed79d57ef7d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/b91d623666e85acba4e37e6bccfc3408efdeb5c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303512.46/warc/e02a9c24025dd32c7bc0d0a3288fe14a5569b3b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/3d7897cbf0d878b93b32a5d622158ec7f38f6723.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/5a8ea91628c5d13aa6ded705a455ffb7f9933322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/5d3792c54f5f9a3da2de9abb5b86415630bee862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/5de4aea0798bd43c36181f46dc21e40822dffaef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/90a832cac061f62ad1491733cd5152294f4c1f5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303709.2/warc/d0bf57cf5ac3e621a8336f5cb64ee2a1f4841e7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/0ff2584001fedde6dc33b4bc502b84702cd20369.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/5b4705337bd34c02728adfdc6bf4e64f4937c51b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/7ed955c06779901b9554db1c9aa9a36684936848.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/987585729d1eca9e25df6c3a112cabf94f8017dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/ab37db57a24af41c66c96fe4a3bb42746e2a7684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/d7afd2668f1ed763130dac191f7b78096747383c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303717.35/warc/efa80466b279d07ee24c9a00dfc865e05a84114d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/11523c54dfc3d1466ca5d355d464fe3534d4a6c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/227348714fc829719b60a391932e35c7a68281d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/294ec8a67bd58d9eb92dd633d34c15062b1c211f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/6fc53db42a1da98a9c17fae82bf62dbc3c038510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/ae7ce870fcdbff249e1b17f9a46739475db6b8c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/de347aa99fcd26a3aebcd3e568d1644b6964b55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303729.69/warc/f98a6007576b66f348ebd7e5928e4bc6523d96a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/6a9d847a09a9da131dcfcaf4c3d35fc97c74bfe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/89724fb192c91e49b816d1559217b4cfdb876fab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/aac7faaf5947238fc9e98729ab3d91ae99a4cf1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/d70655533d807c38b102ee1f89e58d4a3fd8063c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/df2179cb56a539bfa263b0af0b2d7ba4fd58371c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303747.41/warc/ef05eced6d39b9c94edfa2344ade2415991c4a91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/23f03f85621b73101ca2cf738302b62573ad1711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/2ccbfaff739a6e5e017f7ed6ae447ec0b11cd2de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/6a1f2ef189da188888c7e61a07b64f49d4e8fe95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/7083de2315131a2862bce94f0c1c4b6df2ed6ad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/73cd3e5e4448cc15ae2c6ed3e0772569471b38e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303779.65/warc/edad5caf860ac3dce4d43ad3eb1acb78d7439112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/1e1afd55e9fba7976654bceadbb416f21e28107c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/95ac2fd92029844a62181a9c74afb19bb1e1b24f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/988d20fa68d0a95c15f0a75adc6bce734afabb07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/bef26af23d73dab8d528242afdd965f74cd9714a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/f9cabd24df864fb2a6cfc96bb1475ffde6e6d750.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303845.33/warc/fc637c655ad889c535b0ae2ba8e441e70064bfba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/170b0b806a939fc6da97f3b2a44aab6e437a734c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/1d1083b1856b9e46f3704536a140715fc66c0fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/525722746a7eb0ffeea4e4392eda6f23f4f651bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/86acef6b74a8d6b6844b626ac3af5f1679f26f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/9629a9bee9ffa8de226367e95cb0212a1e6cfe15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/cca593e39f779efba4dd7e1ca85de8d378075da2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303864.86/warc/df8a25a0272fc7df59ba467a4c49d6704c83b0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/09c7d1049659438d0e6125b7c7b7b9d5f6406ff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/124c271bb383ff76679352ea92730824a9b5756f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/1434ff01d70afdec4eb976c9dea7249b76c9f794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/5ba665aa8ccfeda97d545da36c1988a2d8151fdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/74518bbf9928d710e367c7ffa075f5ecc410e5f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/92702ca79e6ce91721434d3221e30300cad43f3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303868.98/warc/b84984a21b60563260f418ee35013665fe167d58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/23e4369774d643665fadff4052f0fc04bf31a592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/2ddc1d39da3082df28cdcb6cbaea7129f56f6dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/75b81aa07f4161a6eb99d6ecbc73f67e8215940a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/88158130691b56c9395280c5e75de319b43ccf7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/9f562995311f9684b4cdf0df835c2cc89e21772d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303884.44/warc/bc8e224b822c99e9c5ff2b2452c9d4fef238289e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/0aad7974e23c4ea2b19ccf592b9a8064050652a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/48fb613dc76234c9beac37c8db2dede027afc93c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/6899d4e77163791bfd745fed422ae567108a6814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/c676272e97b7852877e6a2a2d77bfd11cb153885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/cd9f6d5f73d33f83107b007db651e4da7d413be0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303917.24/warc/fb7e4aef195ba25a52772afec4bef167a512b4fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/0bcabd4ad7c150183d85795baa3867c93bd47ae2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/3f76c1ef43b32f331d998c1bec05d4d0e6a707f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/905e73f09d60124238f095aeea008c8a3b8e12ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/a141c031493830ee0897488bae7cb267b2e50c95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/be57ff77f1102c3a4bf65899693b81fd0317af2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320303956.14/warc/de454d669e7711bfbaf26d74cbc18966decd8a7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/5e44743c40e8f4f11c7be37db0d9c81637e5f89d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/6591b6403130473d5f009c6ee5576326626f9421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/b956318fe4b8ae859262a3f38fa83df657794930.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/c0a160a54f589f05f58c663ad20847399378b4fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/c195944606c9976ef84e65327b722a8695149e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/fc8e99e7a1d305100ac07a716154c0f4bd721182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304134.13/warc/fd5d681980e36523465321af88506c69cec391bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/0a6503709d258a86587e7861a5bde55c59bbba24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/55c0e35fd2b6d18c3b7fb7dd6073a3819fdd6b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/6270929d0e486e9808f25c5d217d8451f2482c73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/65a79aa722207f8f06d39897af1a93c1eac938a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/68aa44a16634c4599c6a7216ce110b44aa42c760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/c91cc859bf51ec3339c09dc24db72008a8925217.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304217.55/warc/f5b17ed44a81b2ebb8d302d6d6020e5d4d16e3e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/31caaffe3ddc9cb14da1a169236ff300026d393b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/45fd2f74215fdab96432f7419400582abdc74dd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/5501fb1b6f0db6d959e289c8a345da199074d493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/a47d033bdafabed79520aade9e401f81bbfaec9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/bdeac1dd4665b7ddd6c73330bee07c04842f1961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304261.85/warc/cc537ba8f8cef9c66abaf5d77573d7b55c27bf73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/2bebe6e6d2ead188cc0181978bc18402c2c0236b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/68e7e7c70f4f04118ae056b56d1497b705b8e8a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/c513e7bea81813e79b25f20270941ac9305ea70e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/e1cbe55f304602a4057bc08917551b519f495594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/f9ab61890ac3dc4c4d362501d7fc5a3a6cd9fc6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304287.0/warc/fdf45011efa10553f328d5c001ccdb702ba7e22c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/155282ef89041d150497b8ef03aa0b19acd9748c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/94ed20b32e296e1047bb4a4c70bb520b181a45d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/b2f713ec8ea818e28f5244601b84de564f1f3ac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/b8643c498cc536627c02f9f4f9fdca3ffd9f9674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/c21a6dd353eb90147c572185ccddca1e81e4aa51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.5/warc/fabfbcf09691053dacc95ccdd22860499108708b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/1bb1182e002415c07b1f58d397ab87ccb5321c03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/38b70f2cc31bc267a446a2fece0ac7ff70c91831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/40fe75e64b1166d3e874f43c2e33b38688b72d4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/5240dc2678db219919325190d5da50079f620ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/9e3243cc694d59d7206aa5bf8674a5aa6ee2f8d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/df99ade3220aeea8b5b75895f11c74f41d5ec4d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304309.59/warc/df9f34ab5399d0834e5e6149666f8767822a9b71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/3e7a68843ec33f59d66f9542fe27dd4534cd4dd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/57c9aa52d144870e7d535e817cb70035e3799e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/652b6582ea003762a4d24533c2291c5054cc8358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/6cdcaf100422669a8b8baeefe6f09352873a000a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/6eee95d03f7500afceb27dd0034dcde902e88098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/9b4e3cfb0b38ace2c169d3070c7e5846784a57e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304345.92/warc/bdc6d1086269803d9f20d143eb5633e5f8b99f58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/0af01a30509ac4a502f296db1acf47bc666173c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/0c2d2f96a6780073c9ab0d02dac948fdab14696e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/0d43c8a1a1ca4f1015ff65dd8eac2496a4292274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/346bd5c0bea530d456e39ee9df6e215b40e6b5bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/ba0007a32d1279e65035c1284b4c85350bfbc90b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304471.99/warc/f8057b02412b4646a557abad773d03d679b86560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/383360bcae04a1d924921fdab6cc6e1f07f91ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/4c4bb6ac04997802e363f8ce923056debf8c3156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/7674c0ff49effdadd548258065864e5e5f56d73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/8cba71ebc0b5350c049c6b5d711c6f8ec5e55ff2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/d1b81d02362c04cfffe26d8385555cc2fe0f6f51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304515.74/warc/e6cabcd3b7951d7c5d0d3d9feaedd6866663cec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/502e0e37f43fe44e2e65a1d7a494197326bf2377.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/5e175a739330f07949d5e77dfdf4ceafa5701d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/b1c10dfe6ba09294fede561021ca9bf0e3cf1d18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/d9b3809c9583e156078f5eac8198c002183d1519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/ddf9fd16efc3623986b043fa68044ec922d4db80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304528.78/warc/f0fe37e2013755d48e0dce669ea4611545961e4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/08f7f2bd0844f5af6b724cb82d8663cb973b2b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/3154c6ac8d2f249f8d6a42d51d0626bace9c64a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/3bdb551aae7cd18f8dd15baa6ff0466ae42141cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/588671b1c28432f03a95cacf0d523702d42869c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/890a939ef02bfee9d8b4eea9bc848f3fea70f3cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/8c2cfcd4703b75aff6e913b82af16469865468f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304570.90/warc/94d018d06de11e3e15e49d428f252f34dc3068c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/165501bacf7b2efca9b26ee8d7e642c4402c703a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/41fe24496da85cb65136b7be470d97bef4031e5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/5ed1ebbc00c7eb0c412bb7e3a4f24c1cf618021e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/61942e16b4f85891b1c0556cfe9bd589d205ad36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/6e031a09bd995db66e1c4e0a01aeda1affb15d4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/cc07487f832ededf9e59eca7e04c8246d125ebe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304572.73/warc/f4f74e15fa60f80d17c24adf190fdb0960d10a9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/197a49a82e3112d5d971ee6f476c3d8a4a346113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/30a7503b24991e2c1bb9b7b4c9cb54380683fe7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/57daf15d1bdd64b42410009f136a6e97a4555810.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/8d58c36a0433dd88ac87a30b1f7a6503f3c64c49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/e1b56f4fbefdb858a5cc2357ed4bbb10eb32cd0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304600.9/warc/ec1f082ffcaaf31b123f4322581ce385187e4287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/1e89b2264b9f4dbbc8f0e0b68d77477feaf6c0fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/7778d545e3a4014fade5eba2613599a0a9c972d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/77b0c81ae445fc322b37c92b0d8f72f1c6bbd28e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/7848831fc20a3fdcf52391e9196ab28f5ba2fed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/8e0583361f8a3decb3dafe95022d0cdf4bd35f04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304686.15/warc/f06ec572af4a5b11136271f4a2b36c11ade2707f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/0809a0c5cee89a0cc0fd0f503acf85d6dc88874a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/243f595d5712786460ff8d32eee27a515212ebeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/9e807636edd0574e2036e366ec4697b3262259a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/cddfb9faf3f246e9f9d33fc4b656407b0f192618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/e985c424a77ac365dbae51e77ab99765a1edb290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304749.63/warc/fef3508d0cebf153e33d51f22d4d84961e48f8b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/64447cf20156b9bd46b7d6d0b51953ea8a297e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/74aba389207ad35b1cdbc503848037c7635c577b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/a67e13a0205e8f98536cdf7643399b0b96d484ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/a8e60f8988bbb873632808f611c67847e4f8cf0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/bceb23f83a7f9cd220019e1513d096350b473892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/c6a22eefe13cf1b81b956aee2083f95b0ddab949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304760.30/warc/e9506f823a73e33352f44bef109b8a20fd78ba67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/1558a0df9eaca9624d332dd3e4cd115ce6bc8986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/275f1eaca6818d086dc54ee540ff0bd385fc8d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/313c4ce0f13c15ffe220d8488cbed10fa8338e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/39754442eebfcd13dd31a3f8a3d63079d51d4db2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/7b78c22e103c9dc3bfff678ac6a8bcbe4c5cc104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/9bf0b7f937d8475dccd65f80206b4c072a2a2f21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304798.1/warc/df659a27bba89b00019e21d7d13fd88488f585ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/17a0e663d398c6559127908ee77308c5c96f725a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/6815f020060172cf6234d639a0d1d8abcfd7c9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/737fabc3af0aeebcb640d2af265ad1a8cf62789a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/7ba721228c99f9541ff3a7aab1e4e2750f36d9dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/837b9fc74795d36dabadf682eedb7e58724a8569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304810.95/warc/e2e284f3b6cddf338fba6d0abdcd3bf108c61777.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/0412f0008ca429d6eab362d3b31323c6bdea3091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/1146f5305eadca3f4630cf298e6e4c70adcef845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/215dddaff1eecfeb0a8b96f0559b6e4c55353430.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/687a6ca38e25f1f2e3890ee939542695faad6ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/c4feae52c50b608b462033c24a601b3939c6c56c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304835.96/warc/e002738488673d3bc27ae1183c8c468a38b3d7dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/017c7962ceec45e3f477e2c29e7e07e9779e657a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/04cc0f4264edaf09d4b22bcc5d7c1b9490c35842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/985b488ef7653552c9274f528b13e2e35b16be4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/b22bb6430a2a9699db0df1ca43fc2380316818a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/b6427bcf9621ce3ca1f0d4eb65e7666d43e98d59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304859.70/warc/dd79dfadae69bebd1582c34fc439a48175a4f3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/07e637e9ad65f3598e1102af3a45084c29cb36c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/12f35d6b1ae580347cdadafe79e52c8957a0cb1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/170ceb2c1c8f49ea0f43e82b957f6946cd1df092.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/421970829e4728b2608366452d865faa02ea3662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/ae292e85a938638ed96408f3908818177c827069.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/c30422b3c31e1e7aad7020c32b537b8f0d80f553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304872.21/warc/eb85dd7b83252ca29004dfb9487e102fd91b390b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/0eba6b3ce83289f905b7dbb0179fdfc5960bcade.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/381ea289075467f4c689a82682d209d4abd02a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/4fbb7828ce65566b3e17c90118284b039be05d71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/5043f29e5601e96f41f799f6afbedb4e1dc90c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/82b9752ec47be831ae3a4d470e7dba4737fd302d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/8a9490d3e4d839c74f29209aec0abc9ec5b20272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304876.16/warc/f40b0bec09540f81c4fbd55b9e2240d222bcd4fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/039921206947dce5c794534dbcbb49b48b20cfbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/5fc0722fa02720ac7bc1ee342944da5a4efa7c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/618144f429fdfbcc148c11fd2a8453f92a226139.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/a88e5f9cfb3f32740e28905b081430013998d6c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/b40df5d726d3e1134ac6e5c9af90331e2e8cab8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304883.8/warc/d759b9bba30fd090f003998b16a92466f5347153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/3291555afbe5ef1ee610849066614791e7873047.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/5aeaf4ef040a4499255fba784a1e1621ab0cc0ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/8cba8ce6b577778999855b62a18e133a2d6d4cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/d20556c42173f001214f84b947267288fa006a33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/fb7f141cecad4033c5b58f15e6963582c3a1a34e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304915.53/warc/ff6fdd93bd2dc0e0a5d8e31ec27d979f9c92fafd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/1c3559b82f0c8e3e2105c5414c3baba6de56f369.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/6805b1a42b00b647d611a8f9a461134364ac119f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/a559bd06c9d650d91be51c558ced2756e2588388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/b0cf7ca0cf7125719a4eb65dc008ffc59faabf3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/b2fa38c6d1aad1aa0f1fde75a9d4fef534bf0d2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304928.27/warc/c1b36d615307f0ecddbdce8f6720e2ebbb358afe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/3ffe5633f52bbf78169d41bbafe7bd1ee8971506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/45923bd5358d47c4df442e31a567427373b09510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/4cbb30fd9e70fc59b14914a08af0ef08f993a593.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/5b97f703bdf1d2527220cc9224e9f7bc1212b0e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/5c7cfba0a8cc78fe5d6e87d03960780152edacca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/63028a38e8c74e2032e432f18613e6ad563966f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304947.93/warc/8210ba59e6238a025ec10868fd31c9d979dac5df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/2a96d742789d827e0965f6cd9ebca7f8e7f30832.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/33423779887ae9827e8512ff98638dace206967c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/4993e8de836f825cc35f063e4f8e8928aef7e1a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/82e987eb9d476e1f273cef4e6611d4b4e37c8799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/b6a06189d739fb7fbe4b579735f518528dd0da6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/dc3cc996ab11e8d1c6a5a1f81e802736ddf4f2f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304954.18/warc/ecb0de6ad8e0f9b0b312fbfa2d891b185edb6998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/1a64fa6cb550292ace7bea44d4a3f64668f6b8c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/20d5019ad5f9fae979a32e0f73461cadc55a5c3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/3d3279857e9ac1d26b68baf74856c7f2847a1ac9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/751593a6866c0b952a41524929198552e9b78a1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/8d10e3c6ee8f7a5598b4f6d2a99ff5099556ad4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304959.80/warc/90b1a9948053a60da042ade475693d2dfa10a9c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/002f62fb1c508e20a921e12c3dec256f8f8febe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/169f7a0b1db3ceb2f04bb319ba4e2c9acc96d50c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/59ab0ddf2ba2c6722e7a1999a56679c2c1f45f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/5d5451d6e0a65bf9cd09efc8d6452b6d3a31e3c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/7f21532c190e739291f73a6bfce165073706bb77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320304961.89/warc/a470b9a41337d33860c521316756d66554c8eb26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/490b9fe0c0924079688e6e7cc83650f8e5126b9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/4d1953b8030af60ec07520af93aaaf49585f523c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/5ebd8328f4608ae57bbaa55e9095df3bcd648cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/6bf5db74a0ad2d8b8497f78252e67dd117a34ddd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/784378a00806f6f72238668120fd54988d19d1f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305006.68/warc/85ee6667423fe998a123498229ad3f4b9dc237fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/03e090e91d0cbe79fb69555219a985a1a8cbf596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/259c988f3202eab044f94d1b24e864ec6ac3903d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/37d6d99388ebb3ee786a79ed1611115a85f0f382.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/45577c45327cac9b22b181cf3d532f8d3f7ad68a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/5253ea66dcc1cb16d8d2ff5f9d61679e51211df1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/be2758979ab3818c7566ab4440ee5d2a4a9f64bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305052.56/warc/c0f0e9cb4cc81f9c9170110bdfff08e7d7c328e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/11ad2171b74a094e3fc53a60b0a8a71d25f14cdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/189a8ea66c6694c78684d2ead6cdb5c1bd64a0d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/2f6fbb7751a7129f6143305d231d3e5995de5735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/5e1e2104e779d5d52663a077e30ea6e06f2b12e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/6cc8575b3256a487c084c5cedc9bc6588db81271.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/9da70ec0159796c82ec2b84d506168d553976e26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305141.20/warc/a92238c04b4190c41ab16ebf8881302564b1fdec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/4056fb11473dd5eda6aca84a81339a1cf413e173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/4cce47982c4a72a0bbf4bf97bdfe723734f66ff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/9b55811bf2c310e7708039e15a33748711bbd0e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/bf554a42d8d8149d5fe6b4f2d3f454ca2a8da936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/c5b9a96f7b7cff09ebe2e9635a1117e306d425b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305242.48/warc/fa0155e122a61f0aba96e63348d2d4770566716f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/017b34e56e44ec957faeab2ab9cafb973e22b8a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/0cfe0b5428a211a56da42a4a8dd39bb2c8ef7912.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/bbf00cb5defa4721ef26072c04ee7ba1227db7ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/c77c9bbfd40e4146a673a4a8a2f75ca40c879065.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/c8c851ecdb6824d4a36d6737c1927c62db48f33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305260.61/warc/f80751a3300af4f39bb998c5d0d8e36b7a55bd1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/1e6d8e6cf3c28e8010da90aedf26d120179b6cbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/40d45cf69a16ad009156a954ca8ee048d9856107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/72792c4166e6e9cc214e552d762f7fc859612ebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/8eea3d5b485d139b24ebccec2665d1a6f3a5e7d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/91079ae4dd09bc5d3e89dd4028ccd5b80f99f399.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305266.34/warc/f22706c89f1ce756f0378d2ee74f535237e7aa88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/2fca7571987873507e3d37e0fa6eab122f62c272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/625c3500e037f531732d66cf68a182428062ab2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/62c0b588cbc6dde1484f4c3e36f578ef1f80b423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/88660a1a7f39168e28aa6d3a5a8466964a4f4c80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/96a7bdf75332cc64ec0cbaee40cef4c1fc062086.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/b432259580f86a969dd0a70537ed8a7f404fc47e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305277.88/warc/f2e288042a5a6b0d9ceed0354e102d0e4e416c11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/36b483b0b7662a6aee2f6c82a88d019586b6c112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/3c7a12bcb05ce07b8fbd183ba1ae67995a71b50f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/40e94eaf288a16bb8e29b972f01f194a0c1568fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/50f174eecaa834b2c8ab871f0d5fb8cc6290589b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/54f12c896a887deb23dc2e65ca03864369edf340.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/708b00b17181997782a7bdb738d1af85b796b4d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305288.57/warc/ffcf3aee766ac7d198a0e8337e37e84d7b072005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/499b7ec530a8abb36a4a79725cd010838ff2aaec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/9b080f2fa9a69700b991e548fb08beee6271aa90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/a160367d638634143bd7d1c332d1a5a034483b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/d2277f490cfa4aa7e286f7486944fd5f67b2acce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/d3cee9f4445ac70b6cabb9df042e6168e40f166f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305317.17/warc/dcfdbbfb6e676efbe301f8294cf2e613fe6bef90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/0edb5beb7239c2b5d6c64fdf714ba65c162b9550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/1d0ef19fec93db08aaf3ff3659eaa7d4028c9244.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/2ada04b931af0831d40a143673e514a43e82c7c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/5ff9035ac68565bd84a7ad215ab803378b454f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/75dd6de3d874943bbf9f7b78fb7ab196cbab9bec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305341.76/warc/929bd76c1e235fcaea17c52345162dc227934112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/0eea748863c7aa075aaf5335f8b1fa896f9da90d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/15e5dbf2e9a99c098cecb818bdab9b8604c2f45f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/5ad40f62f7356f3f91d007d659ab056a10dae3d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/6b895ebd9bc2e5a584a0e49446a7edfaf53d375b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/7ff6bc7597eb4c23711074e37c00e66b80a99b29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305420.54/warc/8e9d305790a63484551c5abaacd3670480d1c093.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/1243ed5b881bfe9c5563417004f045dbe15d7d56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/8912ae720db200d526bb4aa79d6573656df32149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/98cd72cec16c29e0b4a2ca7b3b52169f1e037e0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/bda3279b1fc081fe2df88e99085135d6551439a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/c8a23ceafd8d41995565acc805529e0faf538f6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/d7c0ea6e6cb1b13d98a4be4d3a9e1c628df7fa8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305423.58/warc/e8571585abe8f6a9bfa7e83a332b524dda94de39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/1c0d48411a43bfb0c6cea04bc852bae22e86aa85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/7d88cddda806f5a614224e189d32f3120d439f3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/959d1fa36b15fa1f86975d30646cbdba49d33bc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/ac61bc8046abb6970719641799a8e5d1d90db435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/ad3bf0eeb8d9d5a9dbf7519fa6f51e94e193bbf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/aef6588972841562f313b29c6cf2cc73881d1281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320305494.6/warc/b4d822080f5662a90937b1110bb5193d0d50a21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/024873e8ba705bd18c068fe5e5102a6455a2a475.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/54d41ee02703e60a04b6e50b034dc35023b29f11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/5d2335ead8142d94dfe3affe0350ca7941aed7c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/6a54ebb9c5174a1454402a14d9b1c024de1f4363.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/9ceffab68fe6b9ff828ca718916277e8c7de1582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306181.43/warc/fafb0aac8ba519c834eea4b114d5b126db5f1c6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/1f9c94738b2021726d6a8cb04b45abf63b957cf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/4b32841ed1572b778d3ff69d1510def9fd9af27f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/a1f33e05da0ac9a75c4e4202e8ab7e9e4a5eaa4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/ae5e86243249a945b6b97e9361cb381ed6745b05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/ca9f8e84895021502a966e159525d9aa26f2b65b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306301.52/warc/fd0ca87536b2ce0b8f641e1d3cb19ba3024f7dae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/3b6e67d0fce395d38c3219238f6fa6bc72a84df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/69b1ec890bf27db4b1b1f8e765f7fd4e3b704e1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/74d63730f2242f12623fd32dd399bc22beccbd38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/86b10a3f2adfe301307d00f1aa59b4a656be4343.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/aa2cae6eaa5e703f287bacd782af9f8273ea7447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306335.77/warc/ec2df7605554b28c5b2104695a073c79185cc326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/0cada891111be6c7f56d860240795b4d7aafd460.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/448ac08e52283b7ee6896a8051b4071eca19daa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/5b0f1d09f2f7400aae2f4637f191edfb56cd9082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/9a05696958fd2fc898536179a3eb9058e150901d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/cf255c90b0570391f0138889db88860012392d77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/fa9073918c227107f5ec245c1e50da636a19fdda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1642320306346.64/warc/faf1255c8dae214b94255ff46f96dca72304b66a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/18be7e876ffb96b616ac7004b755c26699ece8ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1e1b3257d91799e19871bbec205f88d9fc5c77c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/1f21a86060d48065d8103cc2857bf06ad9052a08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/20052cf3dc260cc21552c2ff695ca746546b48f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/204abfeb56826eb1359c882cc3258b00e62894db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/2372b4a951a2de1bf03d61e4a6eca00327dcdee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/2a53f1415b4f3f9895f8e4b65c8b5baa8d4bae7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/2ea8cf2a593786bef97a984d0ad7003a0c20b78a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/3642c7d72c9c33c8c106903ebeb22457cc0b5d63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/3df5ccd1df8c80cf41015e0639df670f18fa2b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/40b77bf2636ed78f3958779beaa17c3a4f655b51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/46d9aedcc9c090f64580862a540ee8f8727f17dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/48cc8dc1cfb19bb7a0e3f7f5a8c3ecef3090f33e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/49ed749e4c5d1064896a2a923e8e68c1ef38abb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/4c948614cbab2e2ce60417a7e8d197e815d9b442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/4d34099156f9b2d6ece33ad45f92ebff8dc1b2e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5127b5ef477db22b1351d3dc25f57faf6e43eae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/54282e50297b7f2a409d0d43c379ab4f3fc2c17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/54b32e6d084bf8fe4e61292aaa114c369d094889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/578c31c748901d70b040632d42c51071d2115699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/599a8e37a4bc1e6db093b1b687c08b9af352fb2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5b8cacad26b4ac8c5aeaf6782dfe163419d491a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5c422e8972f5201001c4d48d93a2bfdbe8ee562e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5c5bd432a6d7ab617939f6e04bead13b2f55d33c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5cd372087fb660591488b4accb422372aad2abda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5dec7d5591155094f58e5512431b097276dfd90c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5e3c2b4f6f6577a275d1ba335f6bac5951676273.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5ec741ccb0dec5df07fb3ef36ace076b89a751ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/5fcf6a271b0f70164c1ea6f78463206c33f8021e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/67910b25850c22de7b02fa926d9b42350c71dd73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/6a9c991bc56d9997784b6d95036dd46ec53a7b73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/6b52d20f600707d9e5a302d364aa04f54724efbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/71e71b61ff54941be189f947ea25ea8a75bd412a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/76b265509dcc83de2f48a84aab58d1fc3dffce60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/77ba3b1d2ae3468f54c174387f86e5a43ba84b3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/7d76933badeaae4dc7f16dcf9d41491dbd7c74d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/7e23fcfa29afb56e28e0ffa2df573648ebf15d2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/7eb124c4dcb418cca39012f598e0badedda3c560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/81ea646a40f630f07e3b4a5967a6d5a62db25e39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/8264dcafe8579f1269b4b000e14dca861792dc63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/842ffcbfca68527793c16c00a4563fd6c3f73b11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/862861794b967df82b623e4633f8cb23384b227d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/8974125b81a2df7a30afc89aeed1451959a74385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/8b6ae69b51e0baa5f791f05f6e84e56a0a54b27f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/8e2da93b3504260bbdbf34a1f64dbe4c03641457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/9005ba9b5879c170c5ca140eae8151d20ed014f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/926dc7f87f7657837557afbab9d6b30c84e678ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/93778a91a3b1c8b4aa51b89e385234c2da30f7c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/9858b22bcff6d3a3c9bcc2e9db0f7a8bf8cbfba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/9d73a92d3429f5a1c4ea2e60c231bb7d1903e823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/9f1e56fa27a974df0f53995ebc1e48d8c3fead0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/a17122cc8273d0a0df454114b120d77e3bff5b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/a338c3c588d4a139a7b0409ba80226f499142c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/adc6265d6b8211397f1f8f1ac5c93fa8e18a056f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/b0fb091efc3f8b2871adced12b92c4b16077ac83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/bf00a326f88258e3b00faa2930ace11ffafe2a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/c58ef2969e496856dfef62d3c07cf51b3b2bb129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/ca39a37e29179386f8e45405663ec89494d56a9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/ca95f4ce0ff2fcd2ac46eccb293074ab6f1c8b9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/ce2690f35c02dc80fd68a00289d85f9a318aa6c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/cf27487f4325afc26ba070442fe377eebb36ef2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/cfa1ad0f8e431296d04f5a1fc5ace574b4f37aba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/d4f0eb4debf866e12f48f2ffcde799304709080c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/da0a57ae76b9786c337c9dc47f914cfed6427b92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/da8ee588007d9a8395ed19bc2eac4158c6c0607c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/e915a252d70a2da4a9c6796b95c0a5a980df80ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/f2a028fa842aba7df7d2107fa00c8bd1f12e1969.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/f492d62d25d64f2d60c0a6a19d4a8dfa64c2d225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/f56f51c67862ebb8660d118c3fa1528871cfe537.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/f9716917ab90b4202770d3efce5a38ad500b9004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/f9e05072a100dc8af2cdea98f70ff10c019170ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/fa27a789d1d864014d8df479034734a7dfdc54c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/fa8c79bf4f639fe3e9fe0fe89673214c10fc236c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/fb80d68c87dd42e99fc044dc5d59319aef4169f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/fb9ae7206161c9e0a560d53a7db93ed76b90596a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/ff22e2e4ed2b694832819422c4f20a8086363432.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2022-05/mix/input" + output: "/tmp/cccc-CC-MAIN-2022-05/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2023-06.yaml b/configs/cccc/mixer/CC-MAIN-2023-06.yaml new file mode 100644 index 00000000..4995b37c --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2023-06.yaml @@ -0,0 +1,994 @@ + +streams: + - name: cccc-CC-MAIN-2023-06 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/018f55848e027ba4fa534e282ed49e0245cdd075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/03eb8d04471961319f58bbb66ab010ad040ace76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/049c1b2fbb35f711d0d5b24cabe5048d7941d060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/09d1af73aeb4017fba0b3c9135f336617043f72f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/0a7213f430ca4128a494a4cf3f927fd7c837e3e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/0a99aa8a7409303fe9a0b3f0b041e948ce5d792e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/0aa8a1cb354d797c61f126a6a6aeccc88eaf6b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/0d8443926e087b47deca67587edf95cbf297bb85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/0beb6571d4494fb07d760c373fcdcc5415695168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/21bb2b9a341025c65c2b63c82d2fa975556f68d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/2209cc305e43869d2f0e85b7dddf9616d39d642f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/5977c6db8182df69b7143b0982f88a32323abdf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/7e73b140e875dafa2e803c1eb88c3397d635d2b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/82f025787ac2dc1b9f0fa2d99bc07849e606d71a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/8f76a21cab03bab748265d55055d736125c15990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494826.88/warc/ea8db98dc558966e1601264ff55190e76d25457e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/0d8a16a3db501ae38b59c90df34623c928dd4ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/3d5cd6d251bfb52f36196ff130c758b774993eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/5e48a354adccb23f164fdc666b785ac72f220b57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/5e776bafb23e02170c6ec41c1c4ef61e8b635487.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/6bfb4893fb9e895100bb0173abf7e664097bec61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/7aaffdffdefa1757ee9a1607ad9d9bb883d5e83d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/84aa7dec4b64d938ea24e1da468ec6ae8df1e36c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494852.95/warc/ff9cde93e01cee3998fca7b38916fbe574cac5a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/3214536ece5168950e192d6c99e85feb21ffb3d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/37adc19237eb9225ca75437ef2fa5b419b2726e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/4fbefc5b32c6e6de7f78ecda2ab99a6f105b651f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/54ebd9b5b774692ab4c14d99fe0e22b5cec22adf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/798db5cecc9664096e728844914bc5fca7aa33c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/d6a7685d69461d5626633772efea2c7200bb6499.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/de67e3360355c0b4408c3601530d060996a91f68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494936.89/warc/e6e5c3471dcaf879a0a33eb1a32f251a3cf494bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/1ffaabcb6a81fe4218b0c4f38146c89b81ba9c3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/45d9ef13e8ad6db2b7f300d7201c798ca7399480.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/514cbbdefef974514a49190d1eaaf077c3f32060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/888d86fff3ad08756f36d2b99a0c4e5dfc8c46a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/acd9ceea3100cefa6df2ddc435dde57cb0bc7e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/b8ebb966d1d19b41c766b831ed94762c61dcb024.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/f0b3be1718117fe36fa99425039f74dc3fcced96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494974.98/warc/fdddfef94d8021d1ca2faccc2c975c16f19b5e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/12957c4905ec884cc8304ee44c22ea77d6c37d22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/1ae2473cc68e717ff8cb73b6664bfd3103161224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/4a8ebbc52b13e4769af20d05598c7a4594dbaf87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/5524fe16647ad0cf69a609860a757a223efe68ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/70f8425ee1a3b968382e5f6b2debc5d3ef0fafd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/8ef3d37e359f5a39672696386fad7260ce3139f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/929718a22ed70d8ac03040b6f87fe8759c450e2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494976.72/warc/bf6625e959b155188e8d3b2b1f5f00f31c0fe4d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/2f8412db2d65afb9b236a029be1c3dfb29528916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/463896723f198acaf8191cb96476b7e59f8d3434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/497b7b1d0bc9d45a43328a2556f4d1331a777256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/9feb70874d59576b06fc5d4caa1f689e8250c158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/c561be67abf99578e97c61bacbb29ead6286dc98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/d04f459c3b682dfdfe642b19cddbe90f9e328dd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/d5c1d600427103f3132a4d957496011c39937ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764494986.94/warc/e733c9923ac67f6435a2bb6ad15afa958759b7a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/2ec08b041812bca61933f73f61ec6f3fe03e1a1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/af19b97248e6cb4ab61250a51fac7eadff6e043f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/b064b7578fca8a1e4de4b88eaee2d9fe79adfbe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/b136b87684051a370ecec179c6a8bc5225ec5fc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/cdc8778fe03c4e3933ef299526ed00f512e469d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/cece3a172c7cc2e0686e175e16dc8e8e5881b4e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/ee012853c6840eeb037eb94a491928a6715bb63d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495001.99/warc/f5b5e39fbd8d8f666380f24012072abbae3bde5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/1db847a3436eafe1141253c0aa2deb261661c606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/4d18e9c9366f7aa159ade193841c36b953b9aab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/4ddf863aee65e3ab6e0078447263a468cc0ed4a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/54a2ff2a01b3236fe9a7925f380faac25aca1ce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/addff1351114c36e782816a90766c0a309620861.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/c78373f80059cc65bacef4b5691fcce3afe91cd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/dae28bd4827630bf401dab98c20b17ff5d8480b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764495012.84/warc/ef1d0c013dd5354dfb58dc38e4e8a11c2b3e24ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/2ebc85ed9db09bcd865f04fc0abec98663af53c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/375402a82638bf1f44a41fc0686dd9e36a11eb93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/39f90850b415fedebfb5fb00b10ba1ec65e3a0ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/56d27e86afdc294f980144d88f8b756f2206c792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/7f6c44e7b9652f5717caefd9154c6342fe303d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/90b7a254c59fec9e3c2455bea205e846e8c62d8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/b610c719ece2716a841b579773292afe3d819fb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499468.22/warc/dbd1be0b2ff2fd3370ce0f52db0d1a2e3ef4c5f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/0ac35c5ba3c7c62ae75c02644ad89549ce1724cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/36d5e3adce05ef2de035cb64ab2c3fdf022a7e27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/3dc56f2ad2103b687f0862f65d95d859804246f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/54113acf016f8e0740bb2525772e1266902c10d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/58d2e9606eff5073e7f178a821ee6cd4625bcb2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/8cdb8e34793bb61556a7d8de5637ad2abc8c0502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/8dadd0fa1bf3e49b76791741a31595eeddb51ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499470.19/warc/a3129f3748d9de8321a38a8a5affe1c73bb0dd2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/1d5462325536114c985c7cf89f2c07a0af3e16f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/2309d0363a58c5cd7eb9f6ee51a23c6474e93e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/60bdc558cb1c9e116fce0e9bebfe6c5f0583cb29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/7a3153e634a82744c96ec97f4fde4a281c2f8b2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/e8e6e749bef71560a4f9f1178f2b6e025ef5a2a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/e91d20b599cf31d9dcb522b77466b367a5741cdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/f48bf239faca6d98e52df523baf45473f484aca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499524.28/warc/f736b9ee2adccae5e8e1e236722810d829dc32c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/2646103e25c246fa289a3e1445c7ecfba774e19d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/28062e4103b83c0a2bc154770386571ea1e89ae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/6156d315f951f8c4e70097cd299f4c8b50780471.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/7a967ce0a6b75849592c2426f2de6370c5b7294e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/e1b0ecb2a1d6405a3528097ff37d6b9a561fa9a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/e60e3120d9898413a039554ec8ca34af065a995d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/e8d27cc303bc2386f26a588b94f5abc035f2ff1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499541.63/warc/fa95afecfdfa9b5ce0fbafb3c8fbd47e517b4345.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/02586e168feadc6d440c49ce8e82d9b90beb3757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/09474b1d2f23511a37bf1a37b27dd203f36059ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/125d99cda68a7aa91912f9ed53be9a15fff83a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/4a2a8919b1d76e3bb5321e212c295fd6eb5e179e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/7ad79929eabb434c7cb73411ca7eb247552ddcf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/82c630b8b0ad6204895c7701ddc86bb23fa2c618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/97d418087f3c94bd97d5c03a96659d8d39f837ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499634.11/warc/f551201e65b679fdc7054d00313e08dfc7f452a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/01c018b90df50fb3053d2c2a59da83d7c946d30e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/1245034123113d3d0350892060bf0e87e24df0d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/1b56bc0e43f3280b4a0f3f9a72ba592832a76352.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/c6859c15a4b220fa818ee3971545893ee8c65617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/ce8b61b3ef1c1f1fb6387816ee1abe752d2be6f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/d836ae39715792c6b8237d84cb3e272176632d73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/e15584682626126e94119567d799f03c0b9ff73f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499646.23/warc/f183ab5108054de1697ab07f648ee35d0bb07984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/103a1e3dea81ee949e9c811bd1c54bf8462cda92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/2147408fd3d08b5bf07daecac05969943ad89624.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/68786d919b5c515a049cecbc7d5dc79ab43eaff4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/6df4858a7105b197263fb38bbacb76e1df13cb67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/8eb6d5cd1f58a43b62fd8ba63a262dc79732c8c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/91e6fceb2a81932ed53dbe94df1e36b201f2def5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/d3159a2fffacfbd5209a6376849297b22584901e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499654.54/warc/d71324573ef3fa464bcdf43522de81fdfbdeeb2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/194ed088c20c0738b9553d10f7f1d8898906b4b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/4f61cd050895ab293b8d67a152d2ef712ba0e806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/7ec4dea52efa34495f1dbd1724e93ee13067552a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/aa003cd0e8fb367fd9e1845660c2c80fd4b5a336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/ae0b7a23bacbd9337b988d4aa1ca93bc0c9e82ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/b6f3ae75c609dd494dc6be74e57ee394cd07fbd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/d74a0676564e7dafe402e3c98fbaacb592383a2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499695.59/warc/dc9e7da8de20ec848df845545f973c6326a2627b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/00da57217f609a641c03f44a1d09536fdbf7ea24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/1a723978a7327a15fb41bc463c654422707131ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/4677b2cdb38f82010b82f6acf15a8df54b9cccee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/5cdd6bcb7a2c5827624c128af4d3ba076cac3c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/9f59f0e94a0a8760fe0f8c05c6a6c76586383e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/a58f468d62cdcff4b36594305fd6ba0580c295ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/ca8dc09da38934dc8d90179b0f021deaf25ae363.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499697.75/warc/e861bb49583e2b5efb118431f9effdb56940e353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/10e5880d9211c24d2ccc11f68cb5611b926c372b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/1aa6bece9eb1b80106a75d4c5786a4a765140384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/735f8f9ea0a9247ac624d401be644344d7a96043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/757ba5d00df48a4088bd5882981f033ec5b61400.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/9b8fe4d9617552239ba1974f36c1730f75c13e88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/9be0e5ceb64204028c347c443b1ff1c8853f5a44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/9f6c27b25c66fbb6dfa715176ba04cc81475d36f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499700.67/warc/f8d2fb65a8fa8d3bc3429c3ab54b9da46b859ca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/09024d524f63c84f0e4cc9d6dc93b6fc8fcfb51a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/4dcafe288e632b40196dc1e6bd75a947183aac8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/5742395a012e8db13df4f8f9859411d37f8ca124.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/636373e3004ebc5d86fd18e92b08016372adf2ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/929b05b800cd5033b5588766f18bba28e165b841.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/b692b76e442e3cdb197882217ba1fca1503f01b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/b6fd698698de1fb5dd684fc31e5cce832b5c6778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499710.49/warc/baa82d2a783925c505218b24a24c3198f34f5d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/10f771c225ecd5c80b0b197896fbd1137827fc35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/282edd453c23aa0d86eaa702dbd80a672ce39e68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/414beb116f56bfc618d1c94dace73834f1c94e08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/56884a4ed8f91319273609b96a21b9a21b705c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/906357177cba0be5c0c8d824aa905fb134e6b80e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/bd69acf347799b440db6ca5ec6f3adc84b3ed6c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/ca3f354b5201d00d2b8f74aca440c87c4f01f8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499713.50/warc/e2f2dd23371a84199f995fb47c6e1124eb4ce670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/30610d234e709edbafdb8f5596350c89e6e14c35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/38913f71ac12c460382ff613afd29e03454ba972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/498ca9ea8e7470b4b2fa82a0177a296788e5b8cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/701a674f5cf850aefd9e64dc6254d9d8dde49cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/74eecd98a91035838588904977b14022dc6993eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/93598c83f38e9a96149189ed9f660772db6b4793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/c9b81963c99127da2f5811552839ebef7c0d8ed6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499744.74/warc/cb8486de1f044cb2f784e80a5d2c61d2c3e03675.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/149cacc46cd3a6fcb8a912c9de45f26a523f80e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/163606b7cb9486e1d7556646e035b3e070f457f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/1976e0db10ef2c42fb4a025b73923d4ab079df94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/6964f5b4718babf9e29931cedb5730fe84dfaceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/8f402425273a95ea5178fadefa23ad4859f54607.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/93ec80f9c8f3567abd7a758c9a411d965c676d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/b84bee2c1129dbd5f00e7c5ededd971990ffc569.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499758.83/warc/f6dc9ddef35629a771448f23bc7113c50c8b7136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/00accbdeafb8868d102dc376b7379f4f489754fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/3344b7eb102b353d2ed3926951ed3a37f46e0628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/5f453cfcdf220e15abb1c6871127a62909c0cf27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/7a15ff3fd0fe5e6a56438de5eab0760de7a25d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/7b96791ffc0a8256e9929daccfea4f6f83cdf4b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/7dd561770216a74f5d2b90502a8e4cdc2052322f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/969f1549666e823adda4e54e2302ffba0e04d631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499768.15/warc/fce4384477726eff3f6ef2158b510427d15c7a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/6eb1ecd27c0524aa0248ecaba8f465b1e948bba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/7ab94651cc830e46350a56050ae7cd5e718585ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/8ddcb7a2795c81a470943c81eb050f9b39df4cb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/aa4910c383cf7eab7954cec970df13b5c7d49560.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/af314aae9667acc1939c360a491d10a7ab9f7392.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/aff4673d4ca9cb1bea2fed04464b502cd6c84824.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/bcd2e7a27a85c7f4a4ca1848f9cc8311bad04acd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499790.41/warc/e8d504389db1bdc84a5593df9f7de2e7d4db0b60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/046daac00219b318860bd5063416875aec328d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/0a4891ea9ad661d2ac8d30afd1542989d875d983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/1c49240d8e00253292ee3c60a6e4407629798bf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/233e6d25d56135a7ad74a8005a7e00d6f7f97256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/90f6e4b65ed643f4f003221a9838c93293091014.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/c17066ab7ea1a431371765774d7951f64fec84ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/e81bde6c52f1a5eff4197345baf15bc41412eda3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499801.40/warc/e9ff2fe22551e97695a911886c2a02043e7d7a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/1e07512803cc21e147e8de1cd605448e1504a33d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/4085a17dccdd99da71e273d08cb35562d9f72182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/4c3bd5e6492ebf6f24fa514a24f389a680c73224.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/7b652a77fe4301496830c7473614855cb7f272f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/bb54059d307de888e7e3d418748e8c569e3e425f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/c364f83803978b4ebf7cc7a9e0b189d82ddfd110.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/cc10eeee46971e3633a19cbc7f60fb74111f976c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499804.60/warc/ebeabfb7022a040d0aa357437da2f6ca434c0514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/31b9daae7bd5bcd6ae5c5173042b80520c239075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/32ee7bdc4da71f460f87ee1eeefd10e49b4b2f4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/5c03c6be6d30359adbcf119c7a0e00b2c9e33e53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/93eef1bfaf97daac149683aceed51cb643a2b5b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/9ce598264d29853a9f8b7ebfecbbb5cea0781277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/bd1001af587646e1b2f6bc7c3219190d4eb86c7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/c6bb5e998a1019995b9a4b09eb0731b69954638b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499816.79/warc/d0876a483ca9af69f2e0d3927e8caba2b5e2990a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/0af9a8b83b1fd0869a05daea47a41a59fc6a86d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/11090ac1fd99bf768cd73ed71a8411d8da52e8cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/13003973052a7bb9241c4715bb714117ccb4bd3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/29fb6edbfb924684d0b45c7b3602110a0e5a3c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/3be9b2b397b0869c08230159aba4dee157e0263d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/53a8b3201f796229372ab6f48e1de32cdca1fbe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/e59f09d6f42792823f13b80b75e494810e698171.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499819.32/warc/e69754b0ffa628381087e7ee6c421585713253be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/0dac1953ed953499d27fe92004cbadfa2da35a0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/11e40cb66eb0ace5a53fdfc010cd95ef7e397485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/392cb50f35b8973ca617da413222ac87584f60b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/474a6f4f1b8285e605a492a113d8650809eba290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/90beeec0a328e78efea61fcdafd2588989fe67d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/cd1a513082ae56408439f322169d020c06259b04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/e05a6af253f9f2c3c857cfb02fac12bad94e53b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499826.71/warc/e4ff6dd771d78a8d0b775b8c9688c150995adaa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/04dc21deaaa08eef5cfbc49a4738185b2fd1f365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/2a7372dc09b954aca02accba08d75ecb06d9d23a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/3ef1e13983a366d300dc05a673e2a38eb3c5be3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/486330f75a1b7c7928745eb944f29050c7acd3a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/50357234a6f6b945035415efc62db9683fc271da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/5c00844d4114b83e5dc6c430624bdd499d0be3f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/b6e415a7a3d0b897ee973b54109fe0a7751daff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499829.29/warc/c5a3423c1e6068ac722cf532b89d37821199e6f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/1074d3431169a999478bbe77b1c8b4bf0d56e51a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/1b3f6bac3bb05012401c617d1f1784b0683e67e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/3bfafd8473b8d132971c5ddbc153f8e08c5d946e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/4cb5ba2c786df6f5096f862b6baf09422fa1b8a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/4e05554394c01e3f6e011758868c8c4e2a75609f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/8146a6e1eb72889f0481bbbadef7a88927a9e0d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/f5986abef2ab86d27bfbc2bd5069c0d9e1fa6779.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499831.97/warc/fbbe26d47baff09d494a01db3a61aa9ba6a88936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/0daf42917a3b4d7426b625a29db8eb8e772490e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/2140dfa929411dd354366caf19fe47b9f190a88b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/69c43b0dc8217a191118f32607dcdda9bc0b6d7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/6f7b06992693d39c065b7793dfc59426459a3f9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/77bc2d8970f63917c157a4779f9a40bd6e41be46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/a74613052d0a3187753010509aa71e51324a22d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/d3130b00b58545c1647487fcbc07bba2c0f7fe60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499842.81/warc/dc3e534f48c458af28f2a03a69eed8a9c992966d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/11389d79a7718769ffa3c8bdcc5d718004697579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/4cebc1bfaccee21d55c5298fc172d139b82d5694.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/4f3575006f37df01285ddc5215d6c39888f6efb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/51cf8f87185b23936696bdda4d2e5768bea9e55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/78069270885a567fbbdcec2c0817a3f2e7b31c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/9c1c09b43c10617521ba1eea5a5f6c1d2d2e7346.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/9e84dae67705465b9d48e8386bf2c26e0ba32b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499845.10/warc/fccc6b6348dba137fc36ee4f3ac8295b8f79751c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/0aac5577116eb4dac03039a62f877039f4924f9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/1d1325505bb15bcbb8d015d489969ee4a7a2943b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/388a2dbcacbb6babc70b09b66c7a5a348730ea91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/5c8325c0835fa69559f4fcddacf3d95017ec0f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/74700a9ac9fb9b8061a438715da6453119e07935.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/9f857daab47c4c647a82161c5869e02e441f74a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/bfa42d704f2a64d3ccfcfeab77f5244b2a076a9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499857.57/warc/f9e20ef03519c4d22868e846cd81d801c2c22afe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/0956f76c39f850460e68ca69e45df841d2dfd970.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/0c060f1c794787d6a2d05248f10758797d880a9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/34b73aea736ca71e8018fd3503c636e7fe8a018d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/359c2846d4a7ab312e616b7534a2c0e917bef4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/61a722dd5a6d9de479ae327c22fc38c2411890f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/978929619303c38fa103eeb2d0172aff10d73f4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/a9bf471c87843621af00299b19f8ab2e2a76bf28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499871.68/warc/e26e18959b8d51f21d4764a46bbd20e1e719e91f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/03ee48e55e27d5ac8c2676c969da6f5d0813d670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/212f942f3ffd796a5adf89edd8653097cf0d15ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/74a4173e543ca44db3ec428c330539e80c66b602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/b318cdbd6d96dd718c111b3df01f5221bfe45a6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/b8351541cd87aca8534fbfd43d72f2279e8f4e1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/d9bed5383bb5245b62e2e32392ee85ff54b79557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/db03cb71fe67be8ae3b0b1553ed9ff10277b0b82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499888.62/warc/eaa2db4f6e4663ed261c0bf033fb8808be814d12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/12974d96ec39d5e4c0e921e5a854dab007ffea39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/28903ef29a623093da4b1f92265a6d382c44b4be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/2c9ab7db19437b381209ccacbaa408e398b85bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/38cf73ef19a78c06b754dd7facf8596d70ca42e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/726a0d904a1c75e76dc7e5a8d891c6e4ecd2a3e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/8da4f033b5ff4bd5495db3b428c71ed8e2c339a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/9a26c6b6da59c23bc84d3677414498b26e19cce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499890.39/warc/c61880f07570a43e534f6ba7476831dc9cff4fb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/2436d153f78fa74c82adbaa8f26cdcbf57cfe8a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/2b74b26ab631465b9e87624adf5e6b54b983789d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/2bcb304ed7537f284f0cb3fed795bdf85851e7e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/377b502e27c3a458cd5a53b36a9c75c25a49613d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/7cb92c32964ecca0ef3178ef5e10754925289c31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/dc031ed44677fb0c607c181f11c45d9876727467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/f741d948ec9fcfd6c0e2b04eb8ae5174efe524df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499891.42/warc/f95ce42e6d5c5bdd4ef30549eb4db3f2c21a431d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/17861282dea7a03da0b67e62ac97c601e46352cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/258cba13606954274f9afedb62823f5c74256dd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/42dc20932f2c3f641c8f41a4df0d7db61f6ebd68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/65d50f7f87f34aec6fb584388d3d741b4f88c021.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/855ca48fed68fe11a3d0db1e1b7c057fad53f49a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/9ca617d3a02fe0213b7cee44bac795f0a12ff93d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/a0cf1efc60ef23b619ffc8907bf01f338de91246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499899.9/warc/fd45e83491dd54c5bf7d85b2d251d06049348748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/0d538a484a88f48d5d8489d85b83a6cbac102376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/1123626266e3217de05efa6aa919deaee567c0fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/1da7cac4fd8823b4341b533405d2cce20508d1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/28ab688a9803392454595ff90ab64804721ac06e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/630753614e0b612afb9b687bbe31f3809f3795c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/71356543f5c2435a230281d6f0e8ade5e37383d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/bf5236d25c63820dca99d4212cfaeabff1de4cf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499911.86/warc/c41725fc125974d037678ee5c504ab52faa632bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/06076057e9838c16861b81f8ae89d75733babd30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/10e26ae2accc5ae1d4cee34c749f05ec59b38f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/14f09dc98fe28091f5cc819c792a1ebe2cfea276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/a1f508439cd4734c682e31d47cf3e1ef09c06591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/a6ab7bc49ae9b674f9a4fbdc1c2a85d79f2db2c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/ad21d1ef01c2d0a30f18ea9c2ac79ac67492c976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/b3966c8f63adbfbe90bd3f00d5b33c22187b2f3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499919.70/warc/f748da617d18a258d567db0e1553d9832c6350eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/41e32d76a0496be83a22565395db61b55631e8f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/524dcff6e95cccccedc6a9ce0c552eaa8d45fe33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/6ff21ddc0de010ae04b19276c7bb2e4ad80308b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/799b0014018c8e19375302982a3dba2fcb6b511c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/7d71631e9a8f304fdfb113dc83063a91ecb645ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/a770a7cca4a5803b5855cdd21a00ae7ab777ea95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/d93eff6c547889e8c6eaac7ec62c060454feea35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499934.48/warc/db62a2fcdbb634a9154ccbdc4041b02f234e3b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/08d7ef21467e45ea8d33d911a8c822e040d1c2d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/3090b65c9fba033eb4409bf70f8244da8ba6acfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/3d7cfff847a4c693c4a409fe81888dc1a255eeba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/43ce935d2014e5f7cb51e1a82f3be90692b3ea5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/954bc99f5b4d91442fb0f22cdf49baa516e83222.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/c3438f94dcc24553bf74be0a8952a5060736635e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/d0adfc4ef1ce8c5447bfc3dd696765893c47e8f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499946.80/warc/f73cba93a59cf120f14bda09becfef00ec6499c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/089c7e4509ec84e4c1872dc448714ff25deaeb43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/34f23fdfbd4b0822876a52b423e09462caf95f66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/5cc7f7b7b9fe013c7f10e4daa070c0e9ca605ff0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/82f98831803e6310b2f21d8264d8ee0701a05bfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/8ace96e2bba0c2924b1d07321ffcbfe0283b6cb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/9d985c1a6f79e92c5cc18b28b6c93ab114bd0d78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/ac7219d2072dbd9f8be4634281524c8a26bc9bc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499949.24/warc/d1e28e91d107bd022890a4f49c54ced2fd320b33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/3327def69a6d600d61c8b5de3de07c4dd17bc928.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/6c08d0c2707bedaa7096b2b110641ef35fb14cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/6d1d4e193e69aa603320b1bc73b091f4b8c0c992.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/7667cedd86b79e51d0be99a35b22829f2d7cf5c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/79aaea82d2f464736ea40e70d887ce9400709596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/af328dfc5769d158d9010deeb368300cc6f73b0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/db66557248ae89319b2a88f6fe8f8dea6a07f978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499953.47/warc/ff9576395280bfe9e42ccbb6fd4e27a48a31b7b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/0875b5ce4ccb3227b1b09e1c52fdda22df94461f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/26770e16cea0276feb44c460454b2ff024b43baf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/3f01258c7496092641115bfa9fa22f221f0406b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/5885d15805a3ff874578a2a28341d665507b6ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/9e2523bc28c74224f18bc2302de1a966b53dd034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/a49b4dc62d61ecfc67da411b518a8c4e0a546b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/c1ab6359a8ee5308a90ff4a6aa595e0574aebe34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499954.21/warc/f0460d62c8e48aec8b3c860b057b289bc70bef17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/508056e5028b9498067ab1fcd2fcab04df3f1dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/5810fb8829603ec8a14541c4a3302f4d378e4442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/6726996c0e86123f5c1ca181deb77f463a71ff71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/723ae2edb40d7e2234509bc640c6387819a55aff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/769791cf42edbb59edf95c59fe94a997b55750b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/8d178eeb6ee06c5ee40e8d7c8326ef0918d14423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/b960dacfae68bf94dbb2e01244cf7f2e9bb29806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499966.43/warc/f2bc75795d1a4a41cf490d34dc83cc129fbc06ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/091dc6a834dfb4d896fd640c1d3ad90987197129.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/45fcc71bb00f2fc99b21e77be592a770fe227820.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/55622ac298d2f781ddcdf002e7e381401ed411c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/56905d33d77f3e23e4581edbbccfbdf131b694d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/65ad19336b7dba2e1aa4084f6cdf4eb3f6062638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/7064f70ca38e5c2ce812ea95da8ef18a4a3e09e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/814d86904423cedb13a9415bf0f314c9d75e7f43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764499967.46/warc/c97b358509c4faa2f1870c2b22378876b11d5e3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/0ab61ee11bda304d46ad307c08ecd0e6c4421b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/24d8039ca26fcd568c872a974bea7471bd1d3456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/356a86092397b2d27ccd9c667c94901c9961c2ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/424948d94bda5fd77ad9ba29f21461f6071c450d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/7371831e346915b0d1fcfeaede9c9ea84189a69b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/8cd6009a699e25563cb60f4a772b4e9999c6ffc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/b1edb965f793b929a911a3b3bfa3af82ae91aaf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500017.27/warc/e960325e84117a9309b8aef5cd2eba9998927344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/2adc29051a726b429df79833ba6e7028f0c29f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/2b1fd7a9367ec16288e16a14dfd2103a5152950d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/6746f2f75c37cb8c1ed79333c1f39740ccee0602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/7afe13bb5ad24cdbb71915935ef869cb503f0b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/912dea92ae6e21adcb71a4a35b52eb72c2f984fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/baba1cf81672f9962b2d0027d322d08e77f88f06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/d5e2f7cef751e71697c6b690a0f2f4bdd762c69f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500028.12/warc/f8a31844f29eaf258f37edba27c0a8b2097da091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/29cba177ee62b36fc3c43953737c3cf8bf10b28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/2a84ea078b820e908108951e810c8088dc077845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/714e887fea3166bad947a37a46ab91dca26d2f7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/7f74ea42ca967bb6b3a83640fb9061b7871e3b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/b461392ad472f9f8dd47368f10aa38ed72065681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/d44840ca0372a745134d46bee4436577d6a7e50e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/e73db322227ebe3821b4a8e0c4de31939f661681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500035.14/warc/ee7b85101b78616fde16b7c53869e791d7fb5902.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/05bf9a42fab9c06f5f977a7fef68c1768b338cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/1cad7d43e734c85b8106073c2c5611add458738b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/314c21e0f550fa2f185e0e54a5fc310f87869867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/80aecf095235a9551f8480c6045cb3d00a30d977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/a9cb3dd7237c0a95ec05d443f547ad3ca5f52bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/c9a24e47e9bf31373444d5d24abba1b71c199397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/ccd831d8e5398e8fe949bd30dea1819fcd5e9a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.18/warc/f11d6413ea615e486c854014b78cb68fd1ffc80f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/6a89042706224ad98f039155094ec5ed3682bfa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/6cd227fcaca1ed0033e39dcbc9120ecd3342e38a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/7d6ab287278ea3a45061b5d9e2cfdad38bf5c821.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/825fb75e3c4640c39f2d519dbc8968db7952785c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/a7976121d555d126d7b7c23ea15f59c67ee99eac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/b83e27e6cbfd9187544348b1040360feed9ad1e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/d26e4be1406ac703ddd92bea64ce682ad0339d2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500041.2/warc/fe076922d9f30f24072db40a20fdb1679305322d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/188e310fb6b76e78f45452c44b2e69c18d83d4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/4836abf777f7300231478ea5df926c02efbf0878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/4f1c74c69b25a35bf772cfef0d7d1ae5988a3018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/9fd984beca485e88705d2e3323430df736fca394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/bf5778431c780dd982ce448ca655fefd39469c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/d9ae925cec5155b627aa0aa71b1a9d4aead8b58a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/ee140111e7803de5a448bf23b9be5b27ba91e18c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500042.8/warc/f9c2d67ed93f8c078074ee0d953af6c787e79d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/74576b02a05deb04da616a342a351b4f5b49fe28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/83bd3ec7939f89f010b242858da856b14e516421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/930eed3ed805e199eefd9c043a7a15f416731e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/9d6f214cc55a8633952a91288b31f1263a7fc7f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/aab0adf9e88b0502d6c5afff8370a13b4f8c137d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/b4063bff0f75a05c2943f4dae7a54f32f9f17d20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/b4c45d06c0d76df896aa1a3548a11c07b8512394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.16/warc/e63d650142e236164059827d3281367c776baab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/27aec3179b7ed08eaf658754b3ee5466fc6ff378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/33977176abea3b5c52fec51a58213ea405d3d9a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/45397f9346105035dfc128901f483a05f7cc35f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/6b2be203f70d7202066cf55013f9a2a29bca1410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/8a5bc426cdc4165a781e3a5d0cd8535648e28818.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/90125263a2d13434f141b5462a0b1baf56683111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/ae22afb6f43e8b5f275015b3a0a9bfd650e9f384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500044.66/warc/be9cdb3bb5d2a9ca9fa7a1dac60fb87b436650e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/185e08e1a193a5edd2845612b2b2755a8fff7c3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/19fd01d3b22b270b1a6181758f8fb87c1e33fcad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/1b6ac009ab46332e680325808cd94856d062ec94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/202cded9b8d9e885782a36c61ae0d1a6df971ed0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/351374291ee1cfe80140fa0ed6e4625fe15d1c47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/6da0dcd252f7a4279ed4cbfaf16dbde8f5f6e05b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/852ecaa35348de8b875edba097ccf425b10562de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500056.55/warc/c493f28da12320ff56c9ad6eee4eb74e917d5ba8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/0c17bb4e77b73aa729d92b5ebda0aa599102f98e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/0f4bd28464a49c44848c50d449a790d391b4d81d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/67ef653987008163595ef9609b5111145b5dd6cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/7235cdec3c7b463b148ef7e55f446b9ddb6e4c0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/a93b5c4edb109d842c69b7883781a907c49f320a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/c1d24e82a12f4a5ae30a0078cc150d7a76641d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/c936ee7abf5d1c98440c6ffdf2f7ef7bdfed5bc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500058.1/warc/d3e01d06ab61d8506d32f35fa9fa8c82d6641c85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/2b3bad66c4f0cf6556f15ad509373b1eb1d4d74f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/3a298e6436dd90a3349b7b8f03b39463a46ee996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/5a3147340af2645bf6628656b6493922878a5bd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/693c21f68ba9e588ae2343839be56c955c59be75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/9a97de8983b7d4ae49223a754d3a074e5bda0a38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/b673b77d31293cc2eff591580200a93d2baef220.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/c68efb7bf015658e12b237af09f452d77cb46f95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500074.73/warc/cb8a8f55c71759e19ac0640a7fcc5f174ed4a1fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/092ea3a448b07bdbdbd1d565c3ddec67cf1d4d54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/3e0208867c1ff0551b4aac8d9e4409ede9387c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/722fc52d7650f021c1a27ddd2d8951a22be1cb79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/893301843f3adfa7ca7a5372bf51b58da49bdbbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/95d77396c69fbe47f6b77a391232c02fec1c028a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/9bd17121d3faeda979b4896568ce01a931f64ad2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/9ece97ca052935a4cc7705bc8cfe2f2004c45976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500076.87/warc/e9a5bc4572738b0263790aa812ef1fbd9f73f7bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/132975dc1b6a0dcbfa4c278d3f0bf9dc21a3332f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/415942a6d657fe4562038f56512855575799b35c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/758525fce31532c6fdf0df093c6f625736b41812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/ae67f84d74b0058ddc8c11601b96d6cf0105a4f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/ba2cbd7f43861740b38d62ea9cdd254fe5d0ed00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/c4edaf93951ee8d92279375325a6933e06ef6ab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/f1523f535782295af68ef753c951a7d629eeb084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500080.82/warc/fc21c6f5be883892581311122821e9024155738d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/01f51a9b3abb870032a3b19e2000f2af7212a5fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/171f44ec0f43edfac68b69b9d33e5bc9fa91b8b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/8c476c830df12838bb100b6ac84fa096fbc33ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/bbacb744893a741a293d652ff47da5512477dd14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/bf3e4f31339c2372dd21b567c0954a796dc986bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/cb41cdb8e9f05146e74facb87ab10fe11d5905d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/dece26d4700c1d637a1d458eae132d3935e2336e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500094.26/warc/fab210ff17d3b5b065a2a50874785e035bf3f5ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/1971c94a039f9df4acd3bc5e56deb5e19628fa50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/2752cf458b2284f44aa9bc66a47127e3b77cd952.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/359ecf243e0815f6e6420477af41e6c3846865f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/414d3e6aeb3d7c9eea00ef6d15b89082288fafbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/9d5d28fc3378db836d1b02933940b165c23c1219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/be7a09b1f3f1fba2968734443076cfc01eea8282.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/efab1aa9e1cd3f7309a4ab910a3ef833a9068ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500095.4/warc/f24cdc8c178e221aecf0914b90b945acd4f4b266.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/257ee85b2c351dac6ce95c4e7a3b104b230c61fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/64d5b6636921e78e4de3aa404cb3569a3ff7a636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/6ded77d3f2dc94250b6443a1e78e0f578d81e5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/7e682e7fa6ce0ba89219a66f72a3de5b7af19ff9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/a037a04334fca8070f4fbaf9da998f088b15af4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/a832e742ef4da3b8e827aa4ae76115c463722b29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/ab8f417d01d53afc2b1a59b5a684b00ab56ae5c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500126.0/warc/f3974e6a2af64916cf2e31049e25498226cabdbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/193191c5c509d48b843a57c71e25189f69ae2510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/287d33859f0b0a1a37c5e2b277aefd04e2751e3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/2b25db99c35bcbe4004152c663943d1a137ddb59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/5d5aa2eb48e76912284eba885eb30b7dec0eead1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/772b88a223d580bc7e1bee02bdd94fdde3adc785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/a8178293c3f3eaeb5ab7bb8888f2987a255b835b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/b48085d903707c64bfa24e70416fc0c3638a3492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500140.36/warc/c5dbef091cf01bad2a1bb3e55e4068ffd22c56f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/3066bbc24a0bcbeac3598a9a2962b813ee72f6f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/7005b1a135fcaceb07200953aa58fe8f7fe0b732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/72b4f771134f4b032cad00062808d1218f4a24c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/85f368b5724afe09c4b848e9cc6b997cc4cff335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/9259d67ebf67e36ef2ba672006088dd1cf138005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/b724da4ba337105c6a2319d5dccb8be6e6469e87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/bb8f9f942239fc3577342d6c4f2f915250880e61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500151.93/warc/e9dc0951ce84b9004802e4ef11602b7de0599f98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/04034b1a4ed9a56b75e1f0e6874d967e5eba000e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/1296db1eff6bb38a33c76599c31436fe00762c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/3f03684ceabcff799c6081db9e77fd71eefb6239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/826aed7622ecd1c27706514483f3076a6d35fce9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/9183670938961f973d788723b6c189d8129f927c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/9a96ef7180e4f8eee211f3170ffbac1e32011b5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/c7363077a6dfaaf0bd97eb734b34ece6c77cba2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500154.33/warc/d288910745b16716ffff468fe5feb3a0c42753a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/19328b821d2450b3d749fd2fb5ab314a0ab374b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/3d447a02b6d897034fd406e20c3ca3247f3adf6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/5da37c5f5bff573233d75cc61908802f9a8dec19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/790ac471dc863d4bcab5670193e6529fbffa6c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/7bcb8cd7ec1e5c0ce82af1110a5ae0266cdd35c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/b038340e729ebcce3d2c0bd41b7afa25af6b4037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/c03cc439062b84bad6a58eb2ae7026b43b2cd168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500158.5/warc/eb07ac438d2b5bc358fb4fa039d6c67d5eed5888.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/057fe6d27fa2fcce6c93f12b1ab77bef2036e4bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/178e6eedb4306bff067e5510590e4c926c74fb14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/343bd806d3d016b653717c6c89a6364f8ee6922a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/3f9262851a3a93891052df7408cf9b7e2acb45e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/5951c929c276512dc64f908c2141f8d6d4191bdd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/770384f11f8d77dfb16799472ee42a4f92a4d60e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/783f0002ed86eee7b850c9afc8c0324fa1a5ba02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500215.91/warc/dc07964b9e44d079943d18812446a1f09277a008.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/0882b686c01cdb5e20f59479da9a6b22be3a2bd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/21589d0bf359f07e36eb0c886252827f7f99b4be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/4004d4dbf3b69b3e92dbde965e9777d760a0ffb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/6a3e0536b541b9df0e532d5d87cfb25cfac71760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/de2f914a23cfc9998727d5421dc1fd687e4d3879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/e577bc47cdfa6edd386cfa109b5360a73b272776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/ea7333ea2509fef1f2c2454243f75b573cd5e06a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500250.51/warc/f5c9b04859ff49eee0c6a48c179cc013a75d35ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/1bb37f21c85d26d242e7444e497a077cc6dad302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/1c2de79f0e3d11c57be5dc24c965bbf423f9dee4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/6fe3d46feeb24839511dbf736434c4b1f1bbaa39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/7f9a783aae9f399a8135218c10aadd8d94e1291c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/9f9004cf7f1a8ba2d743a0cf31fc67d098c5608d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/cd273937ef3a95f07f35fe34fd2757d96d7d3b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/fbbf45f8784184bb185b56901cf8d59c178508fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500251.38/warc/fe5fbebfee5055415f057f23e21441885c6c50fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/31cd48a8deb63577fe6cf326ca0f380d6f8e1ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/545a0108165443b16838a947410f96194d3e9152.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/60d27082373dcaa43d45611052b2e6f81134aaf9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/8162a2d85a2da8b3967ba5608605685efdc4c080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/9cfe7ca3e71d5330f4e58e409455323f7f5482de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/cac396e5e86e792e0db15e754fa01d87bdeeeba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/e30c91d5ae8fe8a1b5386d8f2865cd4862ed6411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500255.78/warc/eed5accb27f9c3c5300ee58e25c0f3d8f9d32333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/1c9384a60a91a1a4455ce0e720ac18d8be587dd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/235b63df211208c51c76c9a75dbefc4bf535735a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/70c46be59f18d59a487ab78c912696c8f18b26a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/aa0c8aef6bde08139552591a5dd0aea22127b7c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/b44aa2bc0ca3647d5e6c81f33ac5fd897aa91e14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/c79cc7d5f8697d7a2240c29182d2cdb6a78af066.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/e74a803b9fd30ec31404dd02d34ff6195c68c277.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500273.30/warc/fc894a79d2761086a74ea9ee1ccb5a9dd937c27c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/56386c9f9db022c12a6b5dc1cfa2739b664cdfc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/965a0a9cdbb40f1e8d619a820426b203161a82cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/9ec28bf08f3fca07a8acc208df762a08aafdfad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/a2ad8c31691e682948386cc32eb0952df5687f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/be5534e7150dc13c5c803ac62f40b1b81b9df891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/c3575b7350b4dfc6d771140a4750d3fc79b99238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/de5601cf03a2a7aeb0a18c23892d914fedf5cc9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500288.69/warc/e67cebf92e4f12ced087ac5448cfbd91fb501070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/297d29540226a40c2f085fa7559b13f89a7e1077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/4712223c1e8b16a2eef0fbb5717aea3960dba814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/906e78d869987218027407b19c773cda131d6f2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/a03fe4e15a015f7a66bc455d7e3d86b2f7547280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/a2f7316a374cb4c6c30e4d389de50dc99de66112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/add47e8d75347b25f29ec52a7d7b11f3c0bb351c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/e05cac8e362eec33e5f0e619219266e14941eafe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500294.64/warc/edc74ef9d6ed255ee205992e7e15e2e6e469f2df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/1c4b1e1c36d14f11c1c373d6f79171edec2a99af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/33343e4e43b13111b53f0639f2c34980378f6d65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/7ad2ad0b1c679088e4eb8ad509d1d30439a127bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/80b788d706e3df388d444d3fd373ad72eb55b804.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/9e2198b4b8c143c222dad415ba2ddee2798d6385.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/a888eaa8916fe5604446cb08fe2d79f13714908b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/cc9f4c267f2b5ec362985574af9520baca76002e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500303.56/warc/df40b64a72aeede5b179803c499558a8aba614ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/06ce2978c496e60df87ec44e4ad94e7944763a63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/3218c9c5f76e53c6199b984accdc2badc136453d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/4b0a0f48f77dd848bcf14c5a70bbf64b861073e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/54f11bc592ed6d5f72eb7659b1d1dd0116466182.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/b3115198e328142fd25a0cab30178b6c31c4cdb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/cb190c46eb18dfc970447a1d5ee9c5ee458f30df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/ea8664935db4dfe152113bd65dd91de9be990d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500304.90/warc/f6810ee96c555b2e83e144b79c84f05d664f1eb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/07d614456f144bc381bcf5ce1dc75beb54f6a189.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/08b84323be0841160ae83d984f6c0d31881bae13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/1ccf2c60e916829b806c55e69e9026ff9e14bb8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/4ee5591cddfa6c5f352f8b61b7b7a1cca9a1626e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/b9395a38c4c6e40f5efa060f48983a2096319e2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/c19bdc0a580e37f5e3484688d653ac83bb27d4e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/ce327dd23ee1530c8a6c988f0daec83f13f13a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500334.35/warc/fc73a0b4d4b0eecbe8e7809b7a3bd001dfad6f59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/1934465bbc52eb3dab26d8e11b1d3ebaac2f5947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/31647328d8b5f90aa84ac0c614003d7f08e5e947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/5958f9a6078c231ad13dafd7d28a34769e7bdebf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/672abc0196b7a06e9e53c0491fc53c57b193fffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/b25e483d4bfb52f8273e94ec038277f1dbca8d75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/e6780062ebcc3b5562cdad4ecd56ca939609c9bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/e6a44dae3ffb0aeb4e326a8107e06117214330d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500339.37/warc/f4bad3ff0c27ee76bb5a8a68b7f48e8802862800.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/198456865d87d18e2e9ba4d8303a9a41d4590486.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/2ae3ef436f951de44683621bbed7e4bd54e917e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/561b61850014db3de230bcd82415c2bc894ed466.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/7edf0fb1c82dca16813bb76bc2913c8133f2888a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/ad5d258bb14a9115571535bde5b9cbf96a01be0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/aee74235894208f2efa5ba0ef491784ca762ad5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/e2feb1ed541727666c3988a2e3e8ca64e9148839.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500356.92/warc/fcc691c168dac581f6a1d6fa5ce1dec087cd2257.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/2009cb4e8dd728ef71bb1d634b949358cb5b16c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/4f5e91c5bca8533753e72db984f44e46de000caa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/5b38609dbc96e33d7c99811efb4f39e30872b162.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/77c14d0e6c0e074c6da908cf63ac4be5a77fcd02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/86ff725a19cb2330c8277caaf769099136ff98a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/a50c9d80f8e85a161da7e9793df9cc46210bb3ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/b68e9aaec2584ccf2edae54b3bc6bf8de8412d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500357.3/warc/da22b146033ed8955d0aae53c31daf4df24304a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/0a741ca30487a17f599b18d04fe8655bc679c0f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/2b57ecb1760d1e989a8a647deae919474ee8da76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/5eeaa2ff96093daa17ae5e462082d69aafe6e1a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/621bef6f4d919b97fd3442cd5e2482613304c782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/662a099230d8a3989f8953f77904aa641c217d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/e83a2db9d880bf5411e044e8023048bc5cff66fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/ea2bd33ebc91bfefaafe7d5e2df33ac2b902c281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500365.52/warc/f36c7d929fa946968e83fd0cbabbec2af7645c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/0aa4a48eb5e749231475aca0b7071d5efd859f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/68dc0d6028550b8ff8f6f169416ca4b4246524fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/9e226b420515c4a19ddfa9b19fce73a0bdf88316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/b0415fe61ba90b437f7814f4748c5afec1951fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/d776526b5e9ba26debfcda04fb50563be73b1302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/dacc0f8bbd9bb154180ce35ab8044738ea22080e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/de2c9bbfd3c30e2142d45cf6aa0317b2d0ee8b3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500368.7/warc/fe7adb151db84a1535f14c5aa233c1171228e0bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/030603a457c354b140148988c1aeae90d5060f00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/0a629e9deb3f210855b6ed31cc6da85197afd739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/1eeb8949ae5398d66e11b308073845e7532591ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/40f8af1b61313a4eadc84c2dc4e2414e873e3617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/4988d74e4ee0a1a590f6c2191df35f88427d79ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/6a3e121f18c9f86f3865cae35b9261e571835cb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/a0876310e563d89b1a00f7e7a8c39d122e2b2a93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500384.17/warc/c92da22c1600de8162694cfa0c5e0f434832a9ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/0dd223591e356e6ceb3bda9af84950038c1c19bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/495ee1040fef0d73d0e542148b5f47599c6e43e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/4a3aaff7a285b2070caaa4c9f98f841f06e3c294.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/610c9c40e0df85f63b7b8aa73a88dc4500783df8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/6ebaffe740e948cf0e95950217ceb8a601f1a333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/8975a484825d5bc9b32a115d3d65238482a36535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/8cca3e90d0a23a493e6679d643cea9a3fdb345ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500392.45/warc/ad8bbf9aa3df0f07ee518dbe55c3d46552773f0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/08534e3c22f7da3d799a06eb38eb679e0aacb5bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/216cdf622e1653241f2e305409465ef3a3fe56e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/4233392c46f828f81271d3f1236b2fceed79bc7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/9a17a929ee7c0ad83e34328117aa7c015ef3f1f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/a00e52e2159935cc90f6e3a385bf535fadbe392b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/a666720f325801c90f6d955dd14e26de2fa0892c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/c602ebac6e109e00c9cf7481592849e66257245a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500456.61/warc/e08cbd8ce795e309ad2a621b3c4116769286f7a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/02c47c36b150f60c131e9a7e97eeb91421783187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/062ad17eb2d969e31aa13d871e1e09a3b5f3b07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/40ac81ac2170cb3132e25e33e07214aacc334efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/501c7fb86629487c428b3fed9256e83988e34a18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/5860dfb375ee14cc4fdc7bbac1fd68f61bc82758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/8bca0a3a124715ecb76fe0a42473bf5da77adebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/a982d7b7894f0a8081a9652d27a6771ec0d72af0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500619.96/warc/ddb2d8f3e7b458e0f96b0b6a8e62a445672c8dec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/4f59c9fe0fea49114b7f37e91a19a8a1b6f01b86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/556b77f6399aef35e575affaf385ff2ba3e033c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/6568de24861fb658b0d211b71e36603af1e0e51a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/89521168d9dd82f7a24ec5ed39057c516ec99321.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/92788b2333090a39fc685e5aca949d3c3f59c985.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/e13196f736b0e0c099ab608319ffcf7f28cc0287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/e2a37f8f2932fa01bfddfe02fbad7d11dad837cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500628.77/warc/efc0c1a6d3e4164f22ff097ba3165ed62d1e8a4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/5ea473ef2127698bfe229ab6b25b361580354ccd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/72e8047fa932c06b0f37ead494dc364149e4c84d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/8a2be08968ee41b0b6f7ffd2a19adbe44bb3d3d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/b8deecf451dc5ff89b43b745ee1332baf6d215a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/c1047f52d94db72dc0fe2757afe5165569ff8bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/de2d8d1f3d479f614fab884b392596fcb10d7ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/dff65b0a615ad991ae6fe93565359e254b18fd80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500641.25/warc/f876160641e0b57052f6746d827e33c529377fac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/7b15a73407cea9823bff773813825ccd21228e2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/7e9881dfe829e0f82f6146a3b934760c63529806.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/b9232cf2476b69cc3e390c2bb0364977a5db8df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/c8bd6599d3b89f47c36ae2f14fd6fd71d71f7302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/cc3f833eff3d0a24c0d34681599b1124a3cc44bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/dc75e9bed842bab5d5716a118e5208fe781dc995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/e58a542fb9d48bab71a28f3fab810eec53fe002f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500664.85/warc/f8445f2b09b95e2ca7a49e972aea8ecbe27f89aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/03d6258ccd4716cc22045929c73843c4e3eca0d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/1dff7474c52b50d66f081848fbe1f3812d3fe564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/36bbd795760b9dccf3ca33c998e1bc58159c4b37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/76578c8397a21e35c8384f28cdf719bb81b9f4c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/9a82a7f0fce2068dddae6526e480bc670b424a28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/c5309751653cdb4f599945983911d689ac28e6c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/f390bb7e21c974234461b646bfda077ff08a0ffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500671.13/warc/f638b1366f769df46059db9b79b8ea6513f95031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/169077be7e96e8e4fdb42c63011c949c3fe4cfe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/25c8d05d117c30b89d54e45a0b4da03a1ba05e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/2961a3bdf9d844ad6ab5ac97226b94291da8c23d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/55ae1f5c858baf83af1ab7eb83fef90e56d92c12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/aec44f9d69718949c9892be9bb6d832049ca91d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/baf65b4c7797caa9d6c1d81cb0e548cfb4fdc076.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/f0a4af5cd7c0fcf309a6083df20e1cd102ddeaf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500719.31/warc/f11be572a5d0a8d21912ab9794899ce8e51dbb5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/2000173dfcfef1a77be8beac36af51feaa80dd0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/3fe3fa3956780ca8a87d39e8aa67696370645335.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/78366988236cd6513df2a3b3ad266fdde74be119.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/cc4a895bf86e6554d26ce13b7ef6d1c53bdf67f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/d97db4933c6b9169430411aca78b2e7b974fbde7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/e09df729c93aad6d0b843bb2369d4e91a4a4b333.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/ea9fc43d8cedf1119739d3991d8f2cb6c448cf27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500758.20/warc/f81e12faf4b9ac0cfbb27d4be93e75d9bdd498eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/2418aa7d1a32f8a316ae848a3e86cecee11bc3c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/295ee5249a32d10efd4ddc989de9f27cc1f3c94e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/4f2bbeb43383c6aae0b6d25c6e822c2bc0f70f73.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/52fe1c99cc0de715d9cd2d4e18b298bd30e507d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/6f0f7696c76289bb74a478350582c5a0381dba45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/b31edbeb1f2066e6ed39be5596feec8b0a1cd906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/eee268735833eef5f31330bbce5e9d4b580c4e01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500813.58/warc/fda7e17b4a96ca881373e3cb4fa453ced4412cf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/0f89c293c86580745719fea71779e2395ec23e1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/17569fe23880c3645fc764c245b87d725cbcad90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/1face8dd16025e30c1e4842f8531a33cbbb36c8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/33c69a8dff9159b9881aaada53219b4ee29552d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/a47b07a569a66a1121a0d14559d674793fc12d11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/b52de89dabc32cc5c6f3923c6e43c07b646938fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/c350b755f6608ce96a30528c672db894fe4fa2e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500837.65/warc/f352cbae55f91ca1506ad093750658b1f4cec0af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/22025c3be286c82420d4a62e9c3d8dc7d50b649c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/36baac9f23c71cc8a4dcd9ee2643affc7140e8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/37b179529a48f5defaf19e5863097c20e1719dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/3848594dd75bdad8d80a567597ed680397b29453.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/9ea9a324e81bdde3f05dbb4079acbb8b32e537ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/d96c35415fd99f1b4cd5cac36f3aac88ade606ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/efe64aee74a1bb043080b044ed901c77dcc8f996.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500904.44/warc/fb846ecf799cb9e8d40eee1e5d1fb7e37648cacb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/157c962b9c6f69375c7fc025ac051838ab47cb5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/355605ec483c4e72d38bbb7881943990b5c4b997.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/3ceb065558877b3b93dbcafce3a1492f3aa87c2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/575421095a0b49b4b4ee72103ba6fb343d85b850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/5805482071d8b75c04136f5e4dc07bc497abbe62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/89c5ea0eef82aa4f5415d1fac34ce78d58796037.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/f29aef8f57285b6bf4102632533bd017d2cdfac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764500983.76/warc/f31f93cb8783b64b060e80b96ce7b499ae9c8a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/1524d1964adc9ac9ad5392d533a4b5d3bd5afde2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/18cbec20742065f00059799b5f4e8b9078966427.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/24681f597a1f13262ef09d05692db2e835f88cba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/3d2b5bc2739441a401aefdb2f8cfb873767bd005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/a6d7ccbf2ac5cc9b59a5a4b29d9da3422018a9de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/d0866f256a14f2ee017c1108a45bc97b01b6a6d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/d86329c751878f31446ae91f0a5417124fc456d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501066.53/warc/fabbddbc298fa938ccad1aaf07bad580c92216f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/19dd7669eb1149932b0e4db4cc146881fa3f81c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/73560adda84f9442314a5a0f5f8085165ceb1cc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/a49c458b631ad40a6404ece26c84f59fb25063b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/a9ac66e20b97d1e7d9fe8366b4ebe780626bec97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/c46938986e4bd05759ecae193c0f3df48454bf03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/d066559d85df4886b6fcd1e11b4b4c4f0d9f8d1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/f2944e38ccf43ea9fb92a29a85c1db0a296ec411.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501407.6/warc/fb6b152552c14c143d0488631ca78745af0f4be2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/8eb17aa8ef0458f7d8a6a377074ade00e390c414.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/b1c1e0e82421e58ffb3ee134d19a344a9d21fae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/b7dd868ae79573bb05635ab36820f8ba03c151e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/d841a84e72d7ddbf3afbba3e8d1abdbd515e4b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/df3d9e1afe94c7cfed13d63728931acf01fb20ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/e61357709950e8752ec0154e2c8f5ad8e8eb1a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/f4385281f7cbd47847fe6e2ccf9fa6ff3b1f1761.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1674764501555.34/warc/fe18ca3f6a3626208a059e130638e36999254ece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/16d003cb0226932ca92f42d19cfc3f89780f0e61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/17b3a993a2063648fc834d408d2a242e494b57e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1988de9567b32da2bdb2b13dca1c36a9f275105a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1a1b4bc8cb4080aa784dc1668d821e29c48ec539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1aa8abb3517000baca0a8d0ad630991d9bdb53ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/1ad29f4de438a08db5d629a876fc8c020d0f2f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/271306c1a49400561c784bbb82c34c4efec48c16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/28f4135f0b647b1664c71a61d3886852c5072db2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/2ee1059152b6a7791d4835be3901dba71d9a3983.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/2f5fbdda0ae54ce790810005840c97ada6c850a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/3104aa44ab763afda88638e7c7ce83437e3134d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/33bc2e14e511efacd8b3d7e7ab51f9c86cc3b94d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/39b757e117763d05c2cbf8d06d7e85891614e4db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/3f249cf7ee2079e32e5c378588265eac5bbe2028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/489cc7ae96fc0acef075eaecd87d8274382fa19d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/4a1c3c7e223cc485c5db4f9463a110304b7a91a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/4a62bbd61fb87f062c2bdbc3b890a72849218df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/51226611155ea0d39dcb69d795072b92a618d5b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/52c4518f1029538ee598679a1029c8aceeea5454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/5e8ab91761070a974944ff5b5defc7a571bbaa19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/69cbd42f76ca7d2a87b4b34d67509a1e4bce0011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/6e5a5542040bfa9dd6effe27c943d78bdea561a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/6e8da8c24cbba3fb13f091db94e81239f96afa05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/6e905a543e5263226e5f969470f7c26921a40311.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/6f66511d6c8e9b20486febd2140ece6a11598b25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/6f8180849b9ab38a7b9443ba93d870a2abb54b19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/75924e8922ad58408416afe9bef2e35d7514d95d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/7685df6c6d213e87d767341f02ea316f6ce28c5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/79e699cf783879c7ea4d8787dcda872850ac31e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/7d40a828747f19b5f23ff97cf2cd71b637c7e8d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/856b055fce98dbd6505d622c854da18ee3011812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/9229f173245e730e668a9e755a02760eb2bda17a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/959e5c652e903dfc3fa7ea8e037a00e47752f07d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/967cb848142868df0c84c24d02892fd4452512d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/9a927237cc00777ad81d98d5099f390801d09550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/9fd9296cade556cd2159ebf075dbf5a18da30a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/a69e0d047d7c248b07e878b82ed37be9f6fc9108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/aad8eec8f60493ab762980f9e9db24e8ddc4a017.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ac6ee4ef701f5de38a97bda9679a1f2c1962c7a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ac827daaf72919dfeb588454d1930963ff1f163a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/acb2798f14c55c3650e29337979e372006aed1ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ae00092af9c08b8caf29dce7905cfc286ebdebd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/b2d30fad28c8e9428c26eabb78395edd52ac7c50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/b3c63eb5f23f07f13586f95978322888371a2371.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/b90cd4292dc7721bb043f2c8cb95ed340f66d361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/bdc951cd0350a704a176d3df73e54521b18d146d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/c26448c1720d1ed6d9663ce05a27dc3b3ed88b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ca0e6bb28580206fed92f1799ef2e929843f513a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/cdf0a73e2991eae321f1bddbd780d2596829d370.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d1f1c4a979622a301e84672ed8b8f8e311f29a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d32aa7a31953b47fc649821ab5f66f7dae4ef1e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d436642be2842d09e9530fb5e26fc0a39ffcd2a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d5183c4f5c6914659a789674fd1cfd447d72e349.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d6a1b6d79b10c14ccef8d9c5385d789b03f6d4f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/d809e279a53245e5b9ac0beb07e68a21b8ea9cb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/da582cbc304f3b5bb37a64e50d36e8c4fe07a457.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/df596c7177f7a5dd346c7caa3b4ba4d19ea63d62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/e2b79f66b98df773e09edc7995fb746e4ceb2ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/e2ef3f68d3c10668a1ef7e08e411a5039c95e14f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/e2fd4c19760b53cdc387ceb3d48f18a8a144e988.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ea39181e3a3c9eb4d9d7abce215d5185e32cf6c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ead6ceed24e5207612d53714bda9287b1c5f9d8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ebfed65523f8855646c9fdaaeea83452a8bf4887.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/edb50a8498ff947d705d1cb27683c65771866f53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/edbfb1c1875cdbb9a77b215e518204b8960e639a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/f18560f9c88e045e0c963157f6702c4a8da40416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/f2b5182da549d2f2087cb0527cd444b2cace2515.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/f67effcc7471d21eba022f828628b55685f0f07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/f714810ed8234626a92e24517efc42c1cddcce08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/f7dfba0e73a8b86d5f101047c20979a13beefdb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/fc0cea8d3dc635bd40bbf4940cf594536125d52b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/ffe9b4abaefa4f4aabd5478b8e524b03949e85f9.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-06 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2023-06/mix/input" + output: "/tmp/cccc-CC-MAIN-2023-06/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2023-14.yaml b/configs/cccc/mixer/CC-MAIN-2023-14.yaml new file mode 100644 index 00000000..ac445f1c --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2023-14.yaml @@ -0,0 +1,914 @@ + +streams: + - name: cccc-CC-MAIN-2023-14 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/07e07a2b2877fab925cc7508c967ae7c1b804c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/2d626a929dc6eeec0acf19107575931e8536dc88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/655bb8bf55678acbc563575e3cd086b61bffd49a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/a1b0580d390fffcadb2249ba542e1257ef856128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/bfb2528ce1306066829ef2a8bf79a79fb2b8c161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/efac43be601786dff11702edf5824abdde183557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/f1b8ac202d071d6ae4fa4b6e363ca23b24b2f5b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943471.24/warc/f68471b10d6059ed1f45ae8d0d210f46b49891be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/0b611d8d4c5ef9350a6ed677df69b50cf17bd7c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/49cb795787d32c334ee28d92db3a8a18530084c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/702c37f4575b996d62121101cbcb8b4bd6d99d09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/7c19e217e87cf1e1c93783780c02e42cea09fa74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/8cec3ad2ebb19b87bb4657024f5c4bf3bc5a0128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/96edd0d94fdc75dc8ff8143caf8929112fd4892b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/ac7a62ce75e11f648e14582862e5d48eda0223d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943483.86/warc/d4f8789c68890c6836fe1fa18d5c2c8266e03f70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/3d9c963d28aa2e12cf9cc994c0fd614155b08c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/40b76356eae0336fa5ae68a4c7b0e8ca03096d7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/4c6fa4fb63684820677ec51f2f77bc54153c6f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/683dd5d1ade091215cc8d292b4c6921429b7b7f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/724d41942b9e95fe5bae7d912fd29fdd92ee25a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/902ce9ce4ce013ee93e9e52375c4fd9eb90131c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/df4edd0eab8a789dfb2476642fd507c43e6d092a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943484.34/warc/ec5b70bf14cdd6c5483a1b4058375e2fa6c85d32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/0131715cf6a98710d300c2f5588f551816050379.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/756c2f91b1004c78aa8b8ceac3e8aa424ba3ae0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/8e1dfacfc4b2a4b77432eeb0bc792887d5f80b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/ae425badc76d85edf4f6d337ac8d2fbf9508af06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/b1563090ac9271344aba19dab22e2f15bd06766a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/d6474fe30a44bb8429c16dac7600764fded8c3f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/f0634ba0d494b8aab8638e677892a875a123935a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943555.25/warc/f355d187398f1ddcd47af34e675d695243275217.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/2366872b016e86bc329b0b228c7dbaebd6713c85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/27741b46945c3218a9e5d8970b6ea7d1d5590851.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/2dec45587609d0fa0a7b5d9f90711ded74b070ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/39c7d692fde1ff56170c149d583331e82ce1ebbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/8b324059d535ed70877e8a9b0a8b3fbab36c4161.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/95091cde46bfaf0c2573011bcc7d2f897ae765f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/b7d4816e2b872a92d4ec25facfb0ed7a281d675f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943562.70/warc/e0f2cb6343f45802dfeed81074db3b99bb496c48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/013bdb0135d5731323d97d2c4fec35b5915f9bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/4051236a8fb91739cd1949a6abdead4c9291bf24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/457e4fd9dea9824ec2d90ff4c8c7592323c9728a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/838f11f12e7e7668d03178b51fb726de18edee00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/8a69460cb7b76322b3554db698a879283491ef4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/a1580eb3def3638b75854ed74e39189ceb1fce92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/bd774f6ded5381380e55374c05876e036efebc69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943589.10/warc/e35eda7f3a7a555ae087e0785b782fc51844973c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/2bd421ebade10253a94a0d03133dfa03155a9d06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/339e006fd80c59269f5d577c97decfb94b93e867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/6c2016f5eedff999b524252d0e26feed8409c479.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/6c39e70bdd4fee2d989b7cab633b1b9483a8031b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/8b038c00a511a87074f16e6626cfa7f2386d296b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/9b27fee39ad431f5a680cc3b50b1d6c361dc181f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/a91f5544b25f11f8df9f08eb1284a10a751924e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943625.81/warc/f26fdeaf87764339177e22aa0f9843d660e6937b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/18b7d8d9eddd092bdfea527718b68fd68a93ade5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/2c4999f808b37b6796b1a85eacc5956d48d12849.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/58b7aa7d092f3915f1887f07804ca9a04bd12740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/b01868d98c25dc2f77d490925061d7f137658f56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/c19b5ffd149de5f066ecdecdda7f7ebd4c05a6e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/d48f3fea39ccfff592fc7df073236f4c1374e4bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/e4e785f672890efade0eb337bcdd23b16dfcf236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943637.3/warc/f145cc872932afd88cbda5efa45f66115e8fe2c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/0e6918c21a32dfa26c8289de5efbcdf2f2afa75d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/2c13d75245a0896d7ffe10488fa4cefb61ce91c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/4bbe9367e382c56409790d96ee5fe9c2a2ac32ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/5bafca1402b48b5d2fcb205f9ddf39e06de5fbd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/7457f17cfab3ece9a2dafd9b28d8e5a5d49a7039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/9158e42f2d82ab3b5f84085ee00e46dcce9fe690.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/9403d76b202b249c21f6eeb7ca51e5da1c859281.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943695.23/warc/df7d85344fe9295909826b9429ed3784c31d4d60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/40f5a4fd8a3f725be6bc4970569ad9be4010b86b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/421da7deada4872e026f97288e6e574ccc7ef8df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/45b40ac5b0078a95190b021a2ed95cc9f99d35da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/7f1cb178607ad370f88afc885f35dd6d01aae927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/867cfab926917bb58b21b53ef5d1c34490ae9004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/e4143cb537449e23a2d49b8ef113ada0eded36be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/fd944e3d1522032ad71e8394005269c30b31e230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943698.79/warc/ffa97a5d8f10a3850fb724c031864298b687683f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/30845e9097e062589c642673ad5ad42e4be26a71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/43ae698e13290b53d00599989cf54d56326427ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/441e14c0e9ea2fbe27f6b4437aebf47390dadea6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/57a2be42f8aa06c4d3a13a624c8a457e0cd67313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/5b237dc6975d699dee1e70ec341d72accf5daf91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/910a3e020db321435613e17a15b4d8bbaf95f857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/95d8e536355622453cadc6fb6b790de904f2077b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943704.21/warc/ca7d2f2b525be25409bdb37117f5b2c6c196e407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/12b255bb34bb7cb2860dea55b9b2ea571b569c25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/2c34b2821fbbf08f54c560fa412041c3f0bbaaae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/52b8d71f620441c32d55f6d28d7172b80bf480cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/5566eb1b271f97719a3878637b1345f61b3265d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/a8d1965db710d49e6592beed45004cb0e67fd23d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/aad23e6911ec8f960afbe01b8d2429e2f5538deb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/cad74c6e17f871a8b50156091d083c4dd78c5ea3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943746.73/warc/f06bcdcda45cfc1615241ea43486c1e4c3653b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/0a1377419ac431608cc5e351f60aca6087a5a83e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/0eace66ad777857e9851c3c5e4dcbe7f44c10dab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/287cdf89269dabcd15b585ecbcc7da9b06148dd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/3f8ae9022b70f5186033441fcf4800ee78ffbc2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/5d0d4b11b5f32b64abed2c0208b40cb46469d647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/7e2ea35faafca256ed90a5506a843e29cda44ddd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/fc25c2a210e4524b309151ff12b7f0a7d21ab94b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943747.51/warc/fd1e479eb3f4665643200d9e9f0d0e7761a41fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/02c096064fcf5bb90e065c9ca400bb5ec9e9ea6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/151ac593d32dbd7df8e83455474bb2fddb46da69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/438ca94da900f39b38c27e2e687096ae20874149.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/5ca2aaf70457ece64ac60bdbbac788004e2a5580.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/642df83bf163332caa5a8b554353fa758d867f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/b48e0cb4c1e63a3e22f4b0cdfd99d105a53f6ddd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/bc3f6770b92889c412e66672361a01dce7da4340.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943749.68/warc/f0d2cc5e4e451a398820fee4c130591db0c00870.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/6e74fa99fc1dc12bdedde88c943e601ef919120b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/70d5bdc17dc519110907cc5acd31135378c56eb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/72eec84209bd13d562ced819754aeb3958c78cbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/88a408f5a0dde8f9f7caa25cd13f0ea4dc462cd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/b57c0387d8a4050509da9c6d70faf8ff2c93c2fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/bd97441c116723a98c7af4b9e25673c7bb2c43be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/c6ed8b926368d8b866de506483bb614e474efa5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943750.71/warc/ee70ff49bd4b0a1714821f0a28e6f3db2558459a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/12f9836261d895cba6ca1bae84794f937030e2b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/435d743a49975d50fad04b022e502f00b2f1dea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/767f4eca5a432355493e4d90b5b637e945e18a25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/97e2d550711330ae72d2583930bb901e4ae971e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/ad9ec5e202c46b0c4aa57e75e3077121fa723e9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/af2416631d2e7c0a63d39837cc54ea82284bd8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/b3ce9a5285531ced3db594ad63dd0c7b536926c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.22/warc/e2efdb081e44db2d8aa1403f79964504fefc6173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/44b830c4e12f5a8a00a4e00d0e732d20d7235a8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/4a46b84dae54ac052ad1fded9c3bcb20e600bb25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/63c5f63b7e894b95e520a7dcff3405f0b77adb81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/67c8902ec888815a893683d28f041e465721c5c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/753ac8d119feea3545aa0c1c4f69515f1cd6a8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/7852bc39dafabed8e2103affc5d9988cdc5d5bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/bdbd160edde5c29caa88f01658c6bccf3b928676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943809.76/warc/f7bdf5c3cf6e5e241a0090b1abc1bd339172b5f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/2598fc45dda94318fa83bf5eaa4830a5b8aa28ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/2ba5fc725859b9944ad0c1f49c24f968b602db52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/4b9ce61529cc2a6f6f00fbf1485c8fcb9be75916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/4f2be116fd65ebae99986ab4e928eb7b73a88ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/5cc1d147a98fd76f88801d202e939be826ccd505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/63afe5c13789b08eed4e448155bbbef4e4b47f8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/7dda76050dfc982d55848e879d7a28a9e8265e38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296943845.78/warc/dbc88835468a65a5b30723d025df86cbb6c956fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/05b2ff7b0c909f3014602b480eddf45a2227b287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/4a389154047ebfe56ca59d455d608d980bcb068c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/57cafef6e4998d1439f1a17f69db67a493905f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/790dd780626accc591ecb6f839564a35615bc557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/960b0407fea9f8493228f39324486d1f12f45a68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/b8449f74454f353f2e2e05392e05978335b1cfdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/dfc2b0e31612fbf441e8ae6e82f784566788758c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.74/warc/ef098614513918981e781b319dc2d2fd9f4b9f26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/001baf1483272294068dfddc222b594b66bcc6f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/480e8e82995844388b39c71e6c5ce209d482947e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/681ae6e12b217e0d577e1a23f68092a98a3f7e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/6d266eb5219a979ac38fbeb6bff0dff36f105fa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/8133c838f5c05a52934097775972e6b40777d5f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/a62284d1733fcbe9ca2f4d2367585f146e770d23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/bc10a25f57f6bbafcfe90aecc630c2bb5948858a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944452.97/warc/e1fd5b2762befea293e0992fbf7e27b767f1d957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/042e8c27555f49d45824fbee7652db5ae9fd3d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/0f86d4eb9ad8f99e28968bfd39e4a0925c50f913.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/4426ce2f8fe1164940acbd1d900298a792010999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/6f3258caf8065e0e0140403fc11084e24908ae81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/8c446ac1c1fb08eaed7875e7d0dbde4a65c8126d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/aea40e0dc152dbaba381ae598bd62a43cc09c364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/cdfbaa6768733492085c9e765c2af4ff9bcc886d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944606.5/warc/dd8dfd038a2e8f424c99090ad739d780bd7f450a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/0f2942b75df94e97a21a5abd1b9abc348c5a39f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/12d807622942956da44a4f281d19d3a90d2548e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/195996f77cb1151b494a540c727386b08b587af0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/427fdd3e79f94123b05f013fb6aeb582a19ba5b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/49e4461332e5e7aa1dd883e37d41042dba6985c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/4d596bd6858840de6f72736bf7e2e0ff439cfe38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/a10eef119ca1a63bf82bbf0b8f240e9545765f14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296944996.49/warc/b58e36df9aa70d00f61ff6a190f89d1f5e70aa2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/11cdc9fb2c4360ae95f28e67d83ef543aba87cd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/49823e2edc5451f2271f1bee2663efc6268cf0c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/7a6d8c91e3474d47c485db52c4f2a14f1760974a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/837c65346eefc5ea707999ab0d4f5eaa6de034fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/84707213d99887c3fde7d8d789e5d71fe72ed322.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/ab289be1da2461631d241726e44c40f760220760.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/df03c9960e2dcf2ef74480143730d5c2272afb9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945030.59/warc/f9d2f68b24f8f7a21e50584acf778d975677eb57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/163a98bed10aa41abc2526beefaaacb5e855401a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/26aeb288e4f28f65257c4f2ca8b3d15cb52bf413.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/42c688f7a4ae1ca20a881d16f40d8f3970b0f365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/6d05b3f5e571a70315d4a6b7de693b4395d05e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/7a227de846d491ab5f7fc0a6bafd40837b03b279.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/8932cbd81a15505f54f4597a95f42f99eb611687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/8ab7796be79e6bf3a5af2a9450f8d244117f3006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945144.17/warc/f512ab090d10eec778a1766511f8cdaf7a1d7f66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/12ad902e29e22542e6f399e1ef395730b453b154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/27056a609acdc39629a27ec33d051b9d1c964fc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/2c2ebccfb5e664c733bc5981ee8b067ad8f6598e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/7d94587106331e050db3370943a9a26b6034bd19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/812a6d333d1283b7192f13f18cfd8da7a73585b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/8d504ee2345705780f4a64bc41e6713fa1c1809f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/a00e6a1b2fa961624bae9bb8fb44bc2ba02a77cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945168.36/warc/b4e01083d2c33d678e128c3d89c520b5310063fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/194bd5353a38386d235050b0cb4e0cebcbba93a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/2fa616a1840bb0ffc189103d04622e661a1dc4d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/373cb439832aef7647e5cb2d1cffacadf13c6d85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/444a2b22bf6afb8a019f182f1399dfa6558f50fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/705b5dd4ef3927a6e7b807cab38d304ac9125efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/74e7669bd627ffd99e9a5c54afa28d6a8381f62c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/a415856b34918fa904902620429a2e2fca8917c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945182.12/warc/bb9c3910a677e2b452b9ae3a2666ef92eec24aa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/20ec367dfd45ca02165a5d0012f9df3fa1cc5669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/4b0e0228cf23207a19cc36ae57f8ea7f5842430a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/4b2ea23b3d359558392963bcb5c5fe1398a9f093.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/86e61ae03756131b6e1351b31cf97698fa2c5cdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/a9e61b2ec6d01dbe70b04983cdad153987c25316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/af0ab375311de1391bb144e4f4c494fb63811d4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/f74d66c4077e70d4b8cce67dcd7dd142f0948a29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945183.40/warc/fd9dad866b9c9918adbecbb7d3c3fc796faacffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/0679d21683b457b881002b4a48da3e8c40e3b3bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/099edef03530b73812cc885a7210125e5f064c88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/1dfc0b81f6ec5181fc6e07264ab23c3c6ba1c130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/3ed2b377d5c023f29f5708a34b9fc420992eac32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/b7547bcaeb6178cfdc229c4db818f2484c31c9eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/cc77796525edff98254850d8086e540f8ac0ff1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/e898414dc0a092d4bb085ceac994d868a04e9856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945218.30/warc/f501f3c6d58526e0a11413f76914a6dca3fe8862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/6083903e9962027625d9c72bb0c000af1654f502.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/870d27a403317a19d58290a1edeabf785a56cb6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/8ddbfda5eeff3af2e92f841cb2573a848e6b2f4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/af33103aca71a1a0d6f06ace442ad5eb68e30565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/d453953f7c97486d16075fe70b4e923611dafdba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/d6e18b000cad69de4071e8c30ca43c4d901158eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/d8cce483f9758928d7f27728f1d055e4b3e27bf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945242.64/warc/da4a3964880739475ddcc5b9522ab62416461504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/0b31910253f1b350084d4aa7c5f126010ecf22d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/15b365d0d4b2dfaf93b184eba04aeab5d37a7767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/9bb2bc74f6bede7d326447c543d249accfb29168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/a0e8c69938015ad830ff12b9ae883cdff7843c09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/c5381ec0761fcc27111f4c853c7180aea8793bbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/e847151f1cd26aec59cf7491c7e3952629ab8390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/f6447064d3fa5d7d40903265c51d1b5b48fc8bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945248.28/warc/fe01852de096923f92abd8a7a1a450d77b8800f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/11c05d00e16e9b4680e71546983d7749f5fb604c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/18d373cd6d13cafeef9fd0a84f442da307dd2964.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/497561b61ba67481585bd0002bdf8c1603a89d13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/71792d04f396112763fe32c66f7b17c17148688f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/79fd0c785aadbc0e807aa5dd24eed61eb8422ca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/bd1b246e89e0eea6f899ecdc2dbd56e8b5fde6a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/d93abb75fbd6cbade0535b3a6193c7095d8f5572.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945279.63/warc/ecb36e73706d77d5b1626eb7b23fe53caafc22f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/065bd9c13185c409b9e291caf34c9560a26ddc05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/0db254370e78b5f09700be0eab147e7237ebfac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/168982588cd27936db4459176a119ea5dd9e2467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/3b78b2df6df95c56b9ba950af67528b48ef40e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/47c5a8bf841b1bded15b62e52ac9e561c69687db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/58f1c043e409ea5bec93cb49ca26c7e866ce3ed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/797cb4abbebcd5291d973623b6291cdd261f8b1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945282.33/warc/e02123b2eae1d8a294e82a5b770d7bd9a4dc2858.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/2630b9d81d6d4603a4ead57392770e648c2da01d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/3297e6f369b18a537547fbda1a274dd438d3094d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/4a2a00aba6d42ae01c91f5399f8c855f32326b20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/584f05876cdb1546baa450785e8f959da34dfe4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/811a1ad31ab08406448cf21ded60716b187cfd47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/87861b870f102afa2e26b01177faf1fe39222d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/aef063877507e11ed13f2ca698f925be96ff91d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945287.43/warc/fa4f55c486c8c2c9b80d768f2b87b8099d8f0819.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/083fc7344460b561b658acb4fb4fcbe870d7abdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/44c1bbf678a25e4904ab30553b3639bf24023178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/46f1fbf3707b261274f620c5416256103bba2f78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/6b04baf632f12cc1a86e9a7450870643475a155b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/725545ce1424279754673984d3794a3d7ffae28f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/8837d06d8623cfabcb84ed7be057aa573f312dff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/ab0bb650c3c1f0a3842628b34f7b40626db41c7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945288.47/warc/da2f1f02091272e9a8915f2596d0a5f905c8ceac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/2173e0def5e22202c31d715d0f0ea8e99506b31c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/3221e5cc7b6363088ccbddff61e3dbed9bfa11eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/484c57bb989a2a7c65c375de2073f3889560fe15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/4d0202dba8e83704e670ac7732dd2d98357dec9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/62be5567f336d315ad1a37cccddc921e4301ca72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/6f1581d53707002d66d7d7b4f884c44b22f74384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/df457459b1a51789865ec11d2b0aef5a3b00cdba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945289.9/warc/dff8bccb3f9a9a1c3ce6ca9d8829f0a3cd836bf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/0753a62e4e897a73fc54f099f4fe92b119a2a07a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/42c5fe9f5206b864c85a663f93eb37ffb3668f30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/46de07566dbe5b978ac1ede543ea315f015fa6e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/4857ed64d1af673f75ef240538f5c9e936af8dc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/5d2cca9c28c040ab49dad6eb56170a075fad5077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/b85def898e2e7f3ee3b93ea279646b5de2885d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/ef97f7a626ba58f4cb82e51df0c6d377279af469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945292.83/warc/f55c7116321bc57ad492e2018cea5bb47db07133.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/039c26740abaede5e539fb5dbce37c9d46256944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/2396002fcdc96c0805d4de42e0203c1eb54a3f16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/80b4521d20a7cda2ce87c4687c5ec69e1a27ada6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/856779da73b23820f2c3bf96947f5732fe112ea4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/a7f4f83fde30eb5ad8ddd9d0decac44d1afdfd78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/bf0c17ef3f3eaa21a49b1186604bb160dd43017b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/c0e54f1c60d4951e1386feaef3e8c6e671fbd23c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945315.31/warc/cf29ab58e6506b764ed6c1c89e741305c9e09f2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/090d41938a927276ec69682580b0ef889f7394a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/a9ae1c5d3c8d61d72832f7b922883ec1d1a203f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/b95bd75b1da88bcd99daf9173bcfb226bc9cd1cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/c07462b8e5712ca4348efadd52aa2756629aa04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/c499399e910d3d1bc7391fc44eac67c02994e94c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/dedeaf2b9ee1a9e37217dbf079ab94cba3a60d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/eb9deaacd446e696f1e4ecfc6c2aa3d79d34a117.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945317.85/warc/ecb5a99f4f0f0698b87094c797121c25e2bdbf93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/271a91ad1d6c5136b1b1d10d0f544f73a351af69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/666ee69ff9867aebaca03a9da9991ebc8926ebf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/80a7a43b03074b84c07bb1980e3924ac433ccb69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/b8a59a0b70a31c4df0baaa2c83c054ebffbfa707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/bee7eb958e2bbad522fef70fe45cc517f512fbc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/c953753839e12492a621b3339241b2a13a906bfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/eb5623db8ac6db84c3dba6bd3aafc9cfc1ab7d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945323.37/warc/f184ced7579b2a8b9f4bd432ce3ad708b9288f2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/0399c1dd5bdc69a2825f2bd51b584b45ae27c9d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/9ec7d97b205cbe22d5e11996049e6c90f31d46cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/a09592f9d2f93124e761e3208b33b435833843f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/bf9e19ffd646c3308511d3c08aa6f999861e5c17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/ce2f5aea9f1e5f3019a98cff4249a81aca18d4d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/d3e8d9b069a5301aa970b58cded2ab4188ecd21e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/d4ec5bbfbd83db616686b71fffe8a252048f1d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945333.53/warc/d4f05ca11b575b9ea7f23a33dceefb1124e40530.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/0fb521e769d9c3e868608cad6f5d97e7e710f6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/11cadc2610f04d19663b524b5f186d2e446b787b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/248855cebfd74671194b386503cabac64d99ef86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/37e3520333e38809af0c80c0701c39a5261c1c0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/7160e93c1b559d4004935ef22a3009985871296c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/cff0a20180ede3c90b348ceb13dc63bc2de6b672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/e459f5031884ca4884faa84a20d489759ef17a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945368.6/warc/fa428feb82cc3a8c65aeed907091d0a79bc147f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/05e03c29acfeecfd56f47e19baacae081cced962.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/06a2dc4c5b9de0bf1b1c418c84859dede3cb0bef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/0726ed81cd6dd85df2a2ec8b060c981000602c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/65f0be0063902bbc64a558b7af5d251aa250d948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/72341a6faffa14d847b4476b6ee42cc770c16f96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/c14e8d09d106b265f69f325019de51fcbedf1ee7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/dbf57a64a3ee43b272e569536bb4f21649bd01ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945372.38/warc/e4ee094033fdd57fbe6f91cd763238a67f65d57d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/0b4e4bc9faa05b73241e054280c7b5665531d49f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/12de6ab9b8d2cf373979724b71a3987d477d282f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/3934bee74abbbaba050d7d648f626efe5f07e799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/6473db96225e954579fe34fbe915c9c96408605d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/90c7eed62ea60fd69f51272cdd425079809a421c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/94ef975c9fa0b17c1cc180933901b95047da0856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/b391e676a7d976e63e22faa5f14ec079b21b425a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945376.29/warc/d0ab4bea087dd0118de92e70a7a5f1850422401b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/6673fe36f746890be98a4f4f3d3bdd389d074908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/8bc616f8514ebe9bda9526719efa511e0becc6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/90c9aba17e55b118f3a573cf2b15f8e37f701292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/97895e8a6f920705f50d731337cda64497f2f0b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/a9c55c704a70e91e81b1a76cfa1dc86265290480.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/bb405b1d0a5d580d0dbabfae518fac40349d1da2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/ca9360a463a4a6088a54b8247215ba3432446354.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945381.91/warc/eee53c8ce8508f8f87ded86ea4a6f4fe3f8b7b9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/10a7df311c1087f51cb1b9b064fb06d6c3c28157.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/14e63e54ba4f6ede0d6c566dbc77dfb6f9b6ce6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/193c4703fdc0728d3c513f74e8c69a7934a90e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/1b814e6968d053ace09d423553bbeda140eefcf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/73a3767c3619e80531ce0bd86070463ebaec67e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/df2819df814f9364633927e294314d85e54a8fcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/e094b96e2e30e082b8f6756972781b6672ab6791.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945433.92/warc/fcd6db8e5c8d54e71b9f137a5c7f1b0ddbb0c782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/294cec619c036a19c736351c4a411d8ebd56a32b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/3ebfd39034168c4dc8ea8643e5615d72cbcca986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/a404f8afc91f99e41c098ff3c7300d7aa9403b23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/b817b370922a75c7e1593e5a53d41c0087ceddfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/b978bcf403e11f880888a302262d335c30393da1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/ce58907390eb2064233ac5fa3a3e4fcac281b04f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/de550d41eafe21796fc9fb5a62536ffc836c4984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945440.67/warc/ebf74e2ad5cfb9a74b6be222b1081934f3d48f92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/09759a11734d2f5d6c4c1e7f9d6db559d18a9cae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/124040eb9fe272b19e1a818d327c1ad9e6341ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/18b7389037ddcf68d59eab56216a3e3c0177eea8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/2ddd6297ba9c2385e1b5b1c24770ed5130c9185f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/9e3ab51a358cf7c84d4eaffe3be5e576b0a88823.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/ac9ef3bd2fadac4c10e59ef774144250eec410cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/af463afbadf6f742a9d9a61c9e900a4724e812de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945472.93/warc/e0960e657e511cb7e7417038faa3235b5a3c64d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/65278705a734214a1d3f5f266bbd96e0a0ee86e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/6ff133aad07f5e614898c3cb4c3f5556da0e550d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/714b695a54655694fd767912d89dc157938846ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/84564bb779a65c09d6eafb9073ac984b96f5cc64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/975774e1cfa4a3f719cc7e456085610edb6bae88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/cc5f2498cddbccbd7cde37d5b88f29072c624a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/db28fc94a492ab40826c8346075cdbc82c4a0522.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296945473.69/warc/fcd91a58fd5a928f502cc71394f0300021473061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/030b1d0707627e996325a28b2789e23e649068dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/0dbf4c394c5745dafbf686fd647b13174b89bad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/2ad9a12a4c3decead8bca6e16632286779aa906a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/4d43eb57313a3c9a950064873399c6259e9705ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/5b024565b552b2e47a027e280041124bc47e637a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/7a61997287eb6e9f2646a9f6a23dee8fb35c9141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/7cfe0e22d1b15cd6c80fea24c5b69a64ee2d6aba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946445.46/warc/bf7b079c72dd50deb1ea7f0002b8676c41312617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/111b01bc8113fcb8572eea9f48f122a4cfb74581.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/64d05656d2667e0d9e8b26147bea66d650ee9e14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/77909a0c2bff690643eb019eb7e7bd8558e5cb5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/8f2f24896dabd92cc2341c3ecd559ce7375c4a0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/c57011f0466f86361562ae2fe1764ea711528d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/d683d661326b086af4b009561fdd7812653f1e52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/d8b107774e58578ed9d51b38fc8f5b1747d0fd11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946535.82/warc/ddfab4c241dc184cf4651ea3fbe36f7409d4d243.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/06af657cd60f96842714313e95b1c535c4e4dc7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/495d105cfb63e45d3a069d14a8a79f0d25f7a69c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/56cefdc01aabf87b0eeccacc6ff0ee3062620f65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/57239fc9d06f54ae60f20dba2e89c3805da0d483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/996e2620c40b11c87b8a5757157b646bf0f42dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/b7a8a48d7a1a4bc65e29b81cc8879e7dd284cc8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/dd236eeae5ba4d3e0f3b80ee7f4fc7d0adbf304f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946584.94/warc/e342236d9d65925c348b0e72565d9ec2e010f3b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/04a1e39e16d1ec0adf63b5b23fd07507c1302e76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/2ff0cca44e2f2fd090b3e7bcd29c805d0490653c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/57ce505aaa35560b22794be47669e0df0a5a840f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/5e402869e610e03ea02ebbf70bb01f0fcff25797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/7779377686dc0472f74f57a4eaa4efb4a1f45eb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/88094f12ab5720dd451213ab9b18888cbb7a0957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/ae9668caafbc1978045f0d06fc60cd4943c01d9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296946637.95/warc/c90cfbe1266cba33ee2683f3a6d0249e3745a957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/4e1c4d2e7052f6a2aeaf7bedced3da588b09ec3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/540e55bd421bc07b97c43c469c711f1fc7b34b45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/892302ae6c9ee541b541ba53ed20fe08b56fe956.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/a939c5ac631a76e32e380cc3e60755fbaa7b6d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/b61185f4f8036aab49b968a453dec4bf6810f10a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/db803c3a6acd787f1734e71a2d3ae6d778772e56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/e3adbf2ee3c21c0ccd5f3754946e7b9f90b2b801.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948609.41/warc/ed191562cfc4812584194e73178d0f617b5e7654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/3c89fa6e6d166638251de1e4cae00fb22e14036c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/4488d44eea3c7eac510fcef620ee2c4357830af9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/4a2b13b0c6569089d9ea7c451bc661f7cab50f91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/4c43fc8be2e9adc1caa040ad7307043809ace8e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/7746ba7618c23321e9a25b870d75dbea4c356b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/7cb096d78aa921200e70d871ccac27773c68f9d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/8970efb32215d01651d87f3a55b503e976f3d77d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948620.60/warc/9f489a55144243c232bd826328ddb3d27e84bdfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/50adde9fc38087aa80cd5e24a37c67c6261d1a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/762e5b3ba6ab07fa661330229316dcb91466b63e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/9f23ef839e53b92c666f39674d44c89f24925edf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/a38aa4c459228cb114810a4faa693dee15f6baa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/b66ffa8383b8b6e73609cff9fcef3816e2c3e403.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/c4817e6c27831a633e877ff9ac94dd315ccf22e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/e75f07c9f447f4659824e002ae4e62902c5a6b77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948632.20/warc/f7d21a2f1a839344648225539f14b34fad27dd52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/055f6a18332d59e55c7510214e3e5c37339dbc03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/2eaa23a9e7b9f21f22a3e1e342bfa4b3205f508e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/70642d8cee3c84cf02adee98b6fa2f8dbf67032b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/775996ac61513ef2a9cc28283ad9cc43c6755b36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/b8b6b7fd15845f8a096042a996e782da73fd560f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/d75d070ff5de65c375ade3a5641140296793f9f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/db87924e472bc59ed74e8101b465325e7acc90fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948673.1/warc/de36192b1f7f6bcf19fd96782e9168d49c1efdcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/2512b6b4bb82e44522ac46bf92aea0ea52b08f46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/3cff729c82d4b8f0aa16c821941a2c43c8dea036.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/439fe83fc682552f612c398bd2762364c0da36a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/724f6922753fdb282e571b7a88e7e2d6cf0ae1dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/870a2284fa8dfa72b6f2151e0d7fbc5e6923609d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/ae59acab1f1956802cf6274c8a49429ce1b33b02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/d60942ed3c8b206b56fd31ea955d54686aabcfab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948684.19/warc/e312b1a53ece5a6441e155d1a82f3561917e9315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/1392437cce4d61c460512b1af8f2f718288a642d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/39f279c14d546b65ded86f4ae4a17e9a692b034c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/3c923a56478d74e843492240f7e6d8d919efb2d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/3df790b1d3b82fe8022a6df66fa2340745299c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/5b9f5e2da84bad5adb7e6481cf29f0826023a513.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/e00263d02797202523032ff002f083f0a70e09b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/e61067f533a5c058864356feb47fdef474b314d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948708.2/warc/f989d309a41f8514716e4f8957861f17295ba481.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/1f38fab8dbd2dd2fa536b9079b8b36359c98248c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/3c012082cec921e8882e4d0c6665e17b3242a3ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/5260c9b00c0fa88db60c48de8af19645c94c4657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/5720e10cdb65e73fe2fd2434b9eecad8da7b444e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/6314d584ce80b9250971b4cbee2f117c5b2c621d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/80cd73a37eed0de02e0af6c53c9300047711a732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/912487b8e8c41bb00998b2797fc598ba3a44c145.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948756.99/warc/ed167149180b752873dad22b7bfd600c5efe8d74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/0c396040d508199c7b23046982bf9132f7728eda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/370090169aeff87464d16de44843e9cad306571e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/3f8520bfcf1009fb3b4276a8953759c3250124f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/55b4fe2efe7033a99d7fcc94ebdd306962aab292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/69214c9fd47aa8b3941d876d9f3e11507517b4a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/876d79b4b452a79dd73203717d834596c950a949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/b9c5e7e27f9277737decbeeb6d4b2f01324ff5e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948765.13/warc/f5b8bd56e75d7edf523a45a998feb4bb2825ba67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/435fb6803512ec01cf6d8f5a7487f64c188def8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/44e14f1b6d74c43004bd4b41649e0b80c31a5785.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/4f1a9cff216f6a4a8b32a1a53862421ceda7a045.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/70ad84e5801eddd1f404af48ea0763b15678fe78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/853004f4a9f58cf3273f32e007689af6fda62259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/893d668d0d6984d8cbcf372d53968134f4293f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/a15c9318245e51368cd0284aef69df7706702e50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948817.15/warc/fe54a83e99fb30faf881b8e8db4996955d3515dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/41bb2481e94fe51b24edefa06e67242c5b46fdd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/62e218e87de851e87d79c86ed98a9f7ab90518db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/6e25cc470c593a2b3af7d68594d75d80fe3189ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/772620c5b493322df7dcba4ced081d5d777517ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/789c11faaf6ca2bad9319eff591dd970e5e63814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/a69d0ed0b785e93cd5faedebfccd7de1d74f53d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/c5484fcb6e8a9fc5583eaa09e2f229171def85d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948858.7/warc/e370988efe07e082c11991b88b1a7bc32f7b3332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/1672abb0d636ca0a5441737e075e79506ee74c99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/21740e9051832e6ce71060319623dec74aea304c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/3dfb92dbba40d02b15e99274769d3e4d895d1c2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/42d732f60716928821bae58b3bef019cfda9864e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/63184a591b087de5425f2a92961e7fd02fa3db7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/6e70a0c719189d54ff9dbc91b304972ae30a11f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/890d5987f6373090702e4da7a42a50ab2d5739de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948867.32/warc/cdaeab50029be43bbb091f694d77f1c88d95ea19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/02b57abcfd7a3b341e693613d5d2a9699d28ae23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/9c3be72b9e5f5c3b88e710a1fac4bf3d72577ad7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/9f76679e5051fb087d7ea0ed893e02dd41df8f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/a393e589c07fcd63005068ad9f67ceeb9fa91afb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/b855e7b2bae2a8f883e12b9f427575a48294d6f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/e69198d8d19f69917fe63d8dd32e0599e3b8264b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/f41151c0b91497be6b711536df1620b64345dd5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948868.90/warc/f7b7e110473e19417165089b4da4b3d755c0e7ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/22b7bbc489456691e88def31c16f0b6f20251a4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/2dfc52c650707bb62f3448618da3eea504e324cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/5b324a7ca7f9f33f048340b81416cb89066990cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/5fa7a22d9082e0b979e1f66d1d32acae5c2000ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/73698ed838517f136b03594f4c8d37e888497f4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/8bed93bdc5d684fb34d0c45d888f1fc6f78e6f65.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/9d24e95e30f493a1f457f29176f86c7b6493b3b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948871.42/warc/b05ef518c446f5044bdf8acfc9fcc9aa530b31d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/0ac4464797f5e4ebb9b121eaed570db33cc0afe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/2bee71e35ca12545fc3cdaa1175c0886dff8f011.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/30079a8e7f3bfec5d12424b2024a9930af9e3976.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/317d2330ded202932889c254779c3b030833fa0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/45b884bd844110ac933c76e237432cd5f5baa591.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/60c8fb637c3ceea01416083ab33550a84406c97e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/8febf0f44306bd843977f664e4ff8e1e2498d3ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948900.50/warc/f8c9471fb860dce5812df04a50a8c2700919acd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/08c1cddba807aa1bea4b9d9cefe2135a285a18e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/22bdb7f5252b6142ed09dfe683485c200320fb6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/3ecb73ceeb7f011bfd89233c41076fbfeadf7b2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/6c9ba5bf4a3ee5965b00426accbb96607d5abffa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/a82f332534a949dfe0739352f21d06087060e1d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/e2c26b953ef7d1672ba40367d4008290733c7c75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/ef1dc40e67b11b87d8d80fb9f17aabd462ce8c51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948932.75/warc/f185eaba7a6d0ff85c153470903fcba4c8569711.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/0002e62b24b629e82fccd41064d20192fb72fa72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/305d8cd7c924f7bb2c4587b5919bee3dc1b1cd95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/615934b994432a1d260666e9224e498889bdd725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/8b703dd9a47840ec002c88b0286f81de448b485e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/a56233e6727d8723fc6f803435732e65184f60fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/b9cbdd316fbb2f609ddcf4172084f6064b693c52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/ce24919582a512f4a118c2e72795c1af7e313130.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948951.4/warc/ed016d7c8bfd735cdb8ce1728d22b5e1ab1a1742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/09c9452a6dcecc85df667f55c8e1ef2fae73fa50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/73304884bc81459d6a8489a8a822c75b28752fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/9428fc766ee4d684860e661aa2bf1a2a912f651f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/a652b038e3847aacaaadc6f9adce754735d14d7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/cf84457dfe0a34142e42e99e087eaa8ef1c5b29b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/d024e88bb8b0f5c73da25599bc870f2c445aa95d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/e3ee77cb18ef12c1e9ffa8f65d0deb897f57565c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948965.80/warc/e95fc7a7340bdaf9afd0838756f3789a2ad5cba6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/0b67b6fa4b2a3a8be995bcb852027d491e40854c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/42029810843cba6d29b212156b239b4b0caeb127.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/592ed810d1125905cd2a5a642f494173ad314df6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/7aa3c324c7d0d43c5e6bd0c1b70707fd387e89ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/8d578a3bd95103cb4d41a82c92892f9531f6e3cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/d0ab5419b99c6caa9be3b07457027fa1b1e56f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/e55348253b86b7128683eabd342c3340c0a14583.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296948976.45/warc/f2f4f2644ea9c62ebcc59b818bd6c107550e8153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/2d925e0134bf14de834e8ccea1e3928de982ff71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/3ffff291fa8d80155e10688fc9ae2dfce28ef84c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/521956ed2f2985e9c71f6728e5f98d1fab7ae7cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/932be5ffa0228db6be4d4e40758d7c619f74852a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/9858d48bd65a85a0147f548309ddda995db30d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/b7bacafdaf8aa807ba05ae00cbbb2d6c8e8a93ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/dd37a81e0ddc98ead533127af2dbc890dfcfd665.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949009.11/warc/e2ff34244bbc4e50191c1bd0a45d15893693960c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/0c204e762a91f6d2ef47a369e3aa90376d186777.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/10f84c9a3036ae372334ec2023fc7890393c804c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/7ca6cb5ccf4d76f9f5b745ace86a1ce0e433d184.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/7e49413c77078db8cd1ae973344bde8545bd40e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/b3266796eecccf8959e420fff2b956080a6afcc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/d7de317038f41c955146c95c3b4c93141eb90958.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/da854284770d4657a13099f684f5515d32631c4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949025.18/warc/dae5eef4738601dc42c88030db58202993e6d03b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/09d55eda6f29ee25b6f9cf12add1f0d6aa086d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/523205b30c297381bffa296265a19b9de29f3a39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/ab0423aec48655aa8d1b5b75d40c897f415677f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/c7f00a34b9cfd1873bf15a75de27c45a32dd5c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/dcecf9d114b0a3fbde29e0f68ab650e9e77a6b0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/eeb0837b9439d01f78772701fb1f6120fe3b8722.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/f0e3477002926676fb92424bacec9f4e232b8168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949035.66/warc/fd1e1071e644bf178b3d24f6e8940581490cf5b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/08f392ba953addb5bf58b8014a39c3ef877046d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/1a29bd989fac150333f6f14b7542bd5190dbd264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/46636113566e5db92927cabd749033fa5965c48a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/8d9e5db2295ee446865612767acd796d273fe6b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/aeb0d97990ecdad9dc64b7c26110589933606489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/d19c3a3833e9744ca5865b0d218f21fe589dd945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/ef9e5c8f543b0941f7c670716fc43caf5d69a090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949093.14/warc/f92fdfcc8c9dff33fc23c3bd457ca98b7dcffe4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/08ac5e1742abcf6bcfa812ce13f3630a8f757758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/476286902928acaa037adf20914fa53e2a5b496b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/5c3811395483494e03564d3f9a3997873e772552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/6844357b13e1b6f44cd11ec4c86c62825f91a3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/6f9fa59dcd60b7c962313c826db67da1f413328e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/783240b2046a72f97bfd158647ca41ee741aaea0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/940c54dfc709991214c8a298fbffe006233552d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949097.61/warc/b17e9c23e562bf70ee82590701b6eca10d7e99e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/06e279985a85a04f5a3fe29de6af868536b008bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/0cea78282fab4afabb34421a5602a829ae46a104.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/4897f7d86b54cbab46872e36e7cfbc30310ddaf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/5cfe6a890d06467f7a4811645b31227f02ab66d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/80bf0baa08483d25fe230a583dc208e9d094c2b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/a242ac8ee4702106f994965c0296d5623a6fc35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/b133292ced96838abcbcd7fda260d513d331bb5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949107.48/warc/ceac018955d1e02c5f5f172f20b23c612af2f096.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/1aba50d980b49288721f1b7e81762c5be34d9594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/26b8cbf0af0b889626d3d05b5153004d4374acd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/37a6f306ae5956847035b634da685837fdf63c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/80e6ab78ebad46b677c1555070569e932b6ea28d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/8d71524ae54e5d98feaa71cb7346ed48014d102b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/bd62b63d523de7305f24b3b29ab74e0ec1ca1b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/d4a101ce1d9e87477b760e65d531140bc758e4b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949181.44/warc/e311c76e2da5820c4ab3c6202e0dc9074fc26d68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/12bd29eaeedd0f4acf80884f04a2761d72e4b4bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/459fe36dec9eb9ad1d9822d8f6227ec67b6ccf72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/74d5f450a32c207eefa24d2ea22224a3be86ee76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/8a61c17d173af70694f5a7ae191c9cba6f620900.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/8d3f0d76edb1d594703208ccf880cc34e356a536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/cbeba177cc3a7409d1dd3e5528b46547faac783a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/e5ba83a39ccc5a3027b345552fbd52e116b8cdd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949331.26/warc/e96fd8850127dec20fe095a20e2fdf7a5761328b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/0d546f31e9bb4c1ac4c7053813b1050b597d3ac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/121ece4792a7698433f79eae7086a4243a5a865d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/1f49076f682f0b348b9bdb6b3af188c8533290f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/23c6a9976e65da06c6310e7647778248775af419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/35e2a81783ebed0af9ec004795b744ab3e094a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/4beccec38953f43d111d68ca8e5378d4913b14f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/92db8fde76b33be2e6d128e6fb2e42b9b8bd2204.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949355.52/warc/c071ae8b2851bb033e6728983fb157b139b59567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/239fcfc8bf3161a3f9899771f0d275d2ef088e3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/617c3cc43b3bc23585820d49a20428988eaf1828.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/7c4334d7816e14fbdf070e106ba5a5aaea0034e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/83b1bad6dc871251aae39b3a71fee5dfc3b0ff39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/878444fbb982b70b91199a6e26c9838e6ba5893b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/92989835db466781694e2d0979215d149b00661a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/ac5ebfc33f6dae026e86ada9ca7ab0b26a55ab2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949387.98/warc/de5d514f0f78646a6d7f5cdfd48e52c957696786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/088819bc3692749c6fa314bc4c2b56bc88b13f53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/16a190382367fe37f5a875ad4e9edd823ea9ad1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/1ded20f61fb69585f667960cd6f54bc908d83b7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/63ff6f41b774f00feb2954dbbe3bc8a56940ee94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/9a1eb756e70f2a6799b7a3f87e36b098cb508f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/9e46cd7ffc33bd572ab4d116a184b654618e13ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/bd88bb2f694199ac5a2cf9d9b94add37d4690be0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949506.62/warc/d8379b2805fa0eed78293190ce1c6ae28ce0f14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/3001f2996a5c79e3bc291a1c98f8ae219fa45c0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/3015a377de32b73cab456301af965ef98224af7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/438a0bbf55816068f3d5453cbde184b747a69546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/438d40a8416d68eec4e6c77b59ae268bc65e6d64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/c3997b0b20c215f9282e51970b9ffe59a3e53d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/d02072ff78defab2d27b1db147a5a6ca43f496cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/f1c1307b69d4b9be5b2b1a51e48dea546a844d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949533.16/warc/ff86a59ac1387c685d8b87c146b1d8ed469c71bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/236b5ba43d5e6819d1581214b3d8c18848b1b3d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/243aa905ad60e7e197af1cdb4f390a1ad7b4e6de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/32201d181d28af7db62c27f38ec8c605012621fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/635bffb5c0759eb8370f36d90053c1d2445015ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/9d3afc91f12ce8db383a64f2767d2985575bf439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/ddda9bd97c4806af05fd8520a7c1c72c88bcb3b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/e013bf0597ef7a02be8c8ff1e283f5a08d01d1ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949573.84/warc/f0d976d8794e49dcb962177a940bcf0756adaa42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/4e329abd4f6797f852b7ed095c77747ffe683e0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/70eae92ea416e8e0876000f08a414d87219d2689.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/9d969e4d325fea8d70dffc65b699a18a2e37ff7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/a17246760bc49543df29dcf5cc4968e319d80305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/b1e3cfb6f4f39f9368ee987eb60b5cf364ec720f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/d0f29c946e9838367ad13a8e9a8d90501623409b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/dc863a6e3fba2d3d46b7770f16389c6e28cd8fe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949598.87/warc/e17a03645184eb4437795b5119c1669cc7cfb549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/277b0ade3dafdb5be29e085e8f69a23266965a2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/41e8dc2997a9ba88afdb26f80d8bc69e9ae07e26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/4bee612d2914dbe0a7fbe27ef55cb4212afe35c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/5826d7d8d00e62158e6b8a9dcf89796fa91ca3e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/8bbcb0fb3ca380d4ec99112d3296d84554ade071.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/8de83eed78add8462da72c0d718282bb39be9248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/948b9db5fc4394c07cdd746866df90061ba2f931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949642.35/warc/caf1888d503a5e8f7fcc1e2141f4e9c0b8a26242.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/1ab3de232a0f82203f6cd3496fa712f832427478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/1ffac2e4a1ee66839f904e08c2b9194a8246d2f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/538864812a6d40302b94d030df6e2b528c6493d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/7e43a45dd48b6d8c1d4586c85c50087c24a98990.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/996d2f64b9bceef5f69c6f07555fc684d57123ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/c21243015aa1bd97e5a7d4e2397079df96bb9c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/e3a5bd9a0f0f0de6dd785b60104f129d20da0dad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949644.27/warc/f363325597eab9c49444226c8728fd4307ae762e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/324331b44622988448d68a37d08c943f2dfff28d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/6b843310b796fd233cf196047ff79cc54c7ccb2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/6fb9f81649668059c27a4a2ce1f60fa348469ad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/82d2586481a38c386d6f218760a81c41d36c59c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/91476fdce1a43a21e7415cdb22994c4681004587.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/9a3d39e6257ac2655f987fd7d07690f8b27786ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/9ec2f70f912047d0e3f14af161d52ad0d77ed2d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949678.39/warc/b645ce5c008535262362ee3317c42884589ddb40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/118d9ea0cad20eb0c9a238e68897c1df2cde4879.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/18c0722eef2487a3e07abe03e40b2c4fc586c15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/3a7fd8df3722cb2faede01847a104d91eee541c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/658b1aefd5b41a18b36282e3c4ee8475f03ec197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/8a83d9d071e7970c52d9a0b2fb6f8ab155fed67f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/c468e287e00255c771379c2d45c9dbe6609217f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/e60441c525e8315d4f6643e9f3e0a4ffb7c3f5c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949689.58/warc/f16221d990ee659522ae774dbe61bc978affcc43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/0bbc828147a3786050e61e6ddea17df451fd0648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/29d18ce91ca3f1214872f704609003172a6aea17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/6cdf1abc0133a14c201ef76175ba48ff0fb6e113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/9db23f75cc8ac5438e943ed1e83e77d4e691e18e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/b78699c0ae7970af207340911dc5739b8614d4f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/dff4cc0d20679342482f79c973b80acfb2b06af1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/e475ddb3f828874c121d9dd304b7afad3bb2949c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949694.55/warc/f9b0a5766e1cef07415a30f90b5af3f479567265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/1519f4e619fec79c6c4daa901702d902b709fca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/3109808c6f8d99007ab6cf14bc3f61df526c290e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/336ee97c7433b35c143fbd3605bf25d9b0f413a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/3daf78580b56b3af24265c6d9d2581eda591526a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/41839b5398f4b08300a5e8c78a213d4a3ab01ca7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/468d7a58e6ab32984cbf2c0ac6399491dcd298fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/70261dd469308a10b85387e1b0f4c9a636aa67a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.0/warc/c47f468eaba3041186d8903af4787008d9cbc72f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/0b29d985dd8c94ac3bfeb2d0a400820caa6dbe4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/196a44e1720b8b8e7874d5b49633437f354a0bf3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/278bda3d02c86a42c517189601b42965977cb4ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/4b4e07aeb053ab00b4dbb0e65ddb7799036927dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/5cb1280035ee87c7789ebe36a790288b3a877853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/930e972990b1ad85c9443f2e0ff3f5afa24a3a38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/9394da0214322181fdd5501876de0a59c28a2c16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949701.56/warc/f2937c30e962bce732a165ea68cee73475f8aacc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/5c3d5a91196acce2f2221d7dd8c1bc5ba817ba6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/88ecf1c648de8fda433e537f64d9589408f9fcd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/a3201af610717d6811b9ca79629792a49696210f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/ab93c8b1ef8b5a49c57ac449547290bc41a81e41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/b3a79a053786e6e9ac04b84ce9ca0e5b567ba046.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/c47bdd88d805b656262a2e04e8bcd3f41fe28bbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/f880a7e8dc88fa390bfc32fdccc2bcee76e82f07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296949958.54/warc/fd112b03aee1f53ee4698af5c6f7a6a380c19030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/4f4652ca8a77ed645cfdffbf9501f4111f077a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/7990086c4e07bd24393ef1d4ded0e6bd8961c582.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/840e3a95d2c35258e229f0da6425e8ff8c14bceb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/8a9c567c4da87f65b13ca600315035de41f5dcaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/960c98ee009c8f897212590a3cf3d9f47685631a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/b8ffa83e172680d4ee59367e3ef42fa65cc07bda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/c8500833c0f4efa123dafe92217df87e8e088cff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950030.57/warc/cd8a1cc9f49ff04a9f23d0da768aeb5606ed228b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/3a9bd5ed49f0748c278d60c1e1ef4048c9b62ec6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/5920b384404162c512ffa8ab5ea24069817293eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/6de7fcbcc53b003f7fa3553a16aadd6521fdf2c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/717b7f04962e51f14d946ee64105597c1cbf4596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/8610da4338a2ceebbadbed31b8e308a0ce8590a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/956af4bfca620c8b197ff16298a6370d6feabee3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/eae45c13a5b33f06e64507c99fea6ff5fb484957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950110.72/warc/f7cd0a28ef1332b086c046ff7947555049357717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/2bf55cecd8264787914c5e4b4377b3749dc0bde7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/5baed48684a1f86cbaaf4d6e9c3fce66f37a4dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/6d8ec7294b039b9b13caa0866ec915acb13a81ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/856f46710c30133041bbee836741accd44e3d9c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/8b1d7436615c017d3702b38085185efbe0080531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/8f96568847f31bc6e5c42b9cc92bec32dbc219de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/b9112c83563857919e7c4e61f9010632878f6431.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950247.65/warc/ed15f6ca2ca399a40fd15c7f786355359c78cb99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/002787df7f8dac1931bfc66c9948faa4ae751aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/1b53e24599bf7d78a81c224c28de40b9dd30e075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/1ce8da47ab601d4db016639302230d47a55324b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/4e847824f40b041451368305a081a4a9b240fdd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/91639e53af2eb02da01e31a96e958228999edc91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/986020cd7425bb4adabacc5809e5a1b5326cdbc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/da851f377ec495f6ebadfa2f5149e7f105ed51f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950363.89/warc/eb12091739e079083021ddd0b12c7ff9961e0f13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/11feb369a24a5b9b9284225235c3c83274334205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/2093065d9ac58ad5805f427b10b25d32f6e9538e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/2a5279afe026ccd79e62b3bc267293cde0e4830e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/3939f407d53b68d163734a6de723d683d93f4940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/62f0e562b8132fb588edb91d46597d3eebe41d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/9462b14dc2861889406d292979b441415e6a447f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/bf773cf6ee9c9eb000ac9283418add52d361f92d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950373.88/warc/f5947362ea3b5cc3e56fd62ff77f75423039ebbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/00cdd7245e3a6cf54555e7b4062a0b7a2f378324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/03ab0d3579db336b338a285bfe83450c07725e64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/19f769840e5ad4143948ef621e03fa3cb8787cab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/24d39b86b57d8729bec719aefc192679134f4a8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/412197559f7af86b4938303d724483293276cea6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/5166321b7537b21379760371feba6c49708e1693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/dee7b1faccb5455f8364372eca4cae80fdcd945c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950383.8/warc/f875bbc7ac4308692ab2a93d0f2c419d4d0405e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/070b6efb3e3ac70c1a3fb18965f26ac35571e2dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/51c46273f14c3064dee818926e0d363f7c76cd24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/6571dfc55ec6a72fec517d6775414e0b0562db12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/6f3693962b8bc60bd23e9921c01f640794e3edbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/79e345cead9dfde3f84cad823a936f2f6d1da67c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/88e621876764471effda5b1e770132b96feb5ba7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/9449b032bdad1fed34e74d95536d23cd0045403a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950422.77/warc/bff1a59e5745f0bbe6671b05d331720f886bf55d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/008a35d7eee41c73d21cf2a532480c5653d2a58d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/0304c79a39e1c429412da6c06d3de5178418ada7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/0e7e7f573e2a984846413adf85ad1a5dd355ecb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/3dfc87519be7eb120d3e370b614b0d48d02e5dcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/485a43a514b133b22f9a356e2d8d22033f28cb04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/500831bffdf87901eee19e7cbec309fc569d36cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/7b09561f45efb6ef43d734219ef2ad9ca41a5697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/1679296950528.96/warc/9ee930be3c2cdbd6b2ddc0afe5d6b0d655690687.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-14 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2023-14/mix/input" + output: "/tmp/cccc-CC-MAIN-2023-14/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2023-23.yaml b/configs/cccc/mixer/CC-MAIN-2023-23.yaml new file mode 100644 index 00000000..2744793c --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2023-23.yaml @@ -0,0 +1,912 @@ + +streams: + - name: cccc-CC-MAIN-2023-23 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/010c15ab3255be92969cefdf9460a9d9af48c9b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/1e8f17b4bce4edaa78f0572b4c3c006139815944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/2404f9d7e5e6d5d5bf51e7fa917046c6d269ec42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/496525dd4a50ac8e5944f90bc9bffad1ecdd0baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/5de9dd9663e488a268022a340d6c26964db70571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/664f5c9c98f11e146aaa5ee3b9618bf7d447c2f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/9074ece84136505ce45cf9582be35f3cd2998613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643388.45/warc/d1ae557cb3e2c4664bc2e5a97e8971e59c628387.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/03c40bf8c4f60d502bb91456888cfdc6f6acee41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/310a421a5ee5db08ee2d7c3bcb6b43d03f67263a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/34567919a736a2ce37d700d1c54d1189ad7a57a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/669d6be52dbae0a14754e25e906633b7631fe259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/6db48b2116827ad996fd1a919dd4cbd7de8b6fb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/9254905b584091cf5432936dd693ef9bb53731a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/cf43c2c5550792f0e3e52fe0528653cedb2f2031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643462.13/warc/ec875c305f5a83d1942db1505b51465ac093bfc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/1d5580eac4052ceb2b81f980ef5ccd84e720e769.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/3598048ec09eabd6601f3ebca859507252fb6a07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/3973e90d5145ea15b69a21e0c42a2230beb3fe3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/7ac1c5b3ebc435ab9dd3c07b3947b6efd96c9f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/98888c8c104d238e682394ee504a7bcbda0cb1e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/a9aebec496005f8abb9204f46301d889e573eb00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/ee6204a5a29f1ba519fc597ef5bf6ec19293a406.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643585.23/warc/f3e4b11723436bf640b5b871a4db48a43289355e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/059e9d53e6fb9454376bd09da7d6539cdb95b1b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/503fe9d56b9204c2ccb6362fcfbef0a5e682a866.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/6425f96619cec367ebde9852985c817b27935638.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/71aa624574a1043031690a8b7eb36b5ef52a98ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/90f6bda744ad4c7bc6cf3777890288b6c8be2a8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/c14872abd92126f1bb28962db5ef0ff0294eb3e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/c328a5c0a18ddd17092c134f416a4b33eb0762f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643663.27/warc/d87a726e39d0efd4139ae0086abd1e07c04ba303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/249b8b422fae9a2933acdda4886dd41ec2757e98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/4b9440c34149e65ae36371e6747c0076b82a8894.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/76e2da41d3435287ec163200b7fbebaf292af827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/911a0c75c733a323156532e96894ff6db35d8494.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/93a5cbe041f10a4e75902609b9d0aaf25a070394.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/94ae565cfab0bad91ed3b31f0f8c5a6c5f654180.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/a5185ade388895eab09eb20601995f7a99748d5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224643784.62/warc/bce43399688555e5bedd194571be5303f954d597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/02fed207b44e2f1f3f71b3ccfb0331fdac93fa1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/13ea15d0c48c1226d4d85943b01a24180df80a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/251e4f47c22fd11d757fa4f99725b2c24113c30b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/2dc66f3e184b74f90431fddcd2cd2386cdeacbcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/88295bb22cf8c941f47d87c9ab16210c05b2ff63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/afce220517ace275bd4f21633bad20cb7c3e2cf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/dd077e1e375b929cedb8e848730a67ff70de5b7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644309.7/warc/fee80c73b06fe9312b9e92729ab58db25cac7f5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/2b058aa0b6ebd3b663868bf9a3014c7316e1d3c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/3a613f225c4830a025209b01c7b5aad01c0a6005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/471f232a3b650d00525724d5d728b0ff467eae3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/77c27971ecd15ff570169342c6a537f4e7a6517c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/a030f7c483091949eee42b306b2450d185bc8228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/b2f5db48b2df5df4173b61e4788d6c7598d7904a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/c125122a5433ce4df268803d57638dc7c357af0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644506.21/warc/c48ee6eaf3407436247bab7d67acc29ca32b7995.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/2635295769896bb20a68e012a6e0c3f2f1672405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/2a4824110cd2a2256a28dcf901081d18cfface9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/37bc1704ba39d5f9d885b457d457f9f8466551c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/3f9066d367d52b5a8cc563a4bbe8a3c371a709f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/6553c2ab0d50b4aa83b2f0abd81d4a61ff02a555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/ade0df96e54e279559bbbe24ef052950571fb6db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/c731d08279287abe05a4b16be0bee669c5c05cb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644571.22/warc/e34315f9e7c0ea13386c6ac767eb1e709160e89f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/0bba12b1dfcadf43f2c14d3e0db7134587222bb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/49a152cb1cbaf867af916eb3046c6f9e04869213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/5b776afcd0f3609d8e849d089dc292eeb26b2337.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/5f944ecaf0a2b09e719da2d85dd08f53daf03163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/8b514896b7e049897eef4069eeeaa364fe518a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/bd42a4923e440fa1b6a57869855e50a806837b8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/e9ec5d914001a777775cd0199b60b8989be8a566.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644574.15/warc/fb4721b6b1075c25aea5d68f265e99ac8d41a2cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/13dc3b3744a13c9bdb937b6ce470b764ea779d47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/1ffd254d7cc4830160c15d620f1e3950678a4963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/47084f851b32ab5c3a3d727f24d05e4767f0edb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/4eef3f1d4049a3ea9b7737be909f02a5feb526d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/6a71ec6975b203b6ec02a224e8d9cabaf947d5bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/7a2c28c7539df490c0511aa9b0c8c7fe13c65f01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/b6df65c01f3b8b3a18f5e8f045ff2b9c040ffbc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644683.18/warc/f7bb9244b1d17b655211c1d94e786faebb53545c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/2081f85d7c73fe068b3ba336b8a249012d5667d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/21e08f64f81e83a333f5eb0a225406c29204b96d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/2c61f77bcd63bea1466da73409fcde65cfdb6494.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/32fcca939da9464a4845a571a2bcca5e51764b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/48f2688d950b9bd59e85936271bf0018d9045d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/4ef8df9a1748fe873151fb0097ca58646906ee02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/825317a3f47f65487db00440e801c176cd5d82a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644817.32/warc/a50c0b6b0468762000130049587db5f13c01b664.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/038631007db7c6f76dfa17849d5ec5a67a52a62d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/279f748c880f5c2b30ea87e8cf758acc005fbcba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/5fe9ec2707b5b6db97944fb92b71b9ef909cb234.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/68a0e3989c87368e3d56a064379f7b9e024239f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/7e5ecc08fb80255de550b0bb5e00a46cb8726bf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/9e5b949d34597262d0db0b77f9670b452ccc43ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/d934e49b5c0634c8e9fe8a3dd381254bcf577331.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644855.6/warc/e9477274dad1b1fb266a4e0af2709fb2c1a86814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/1067af6bb6c36567aea67d91665d09a98a012a3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/2f3417bab1b18565ffd77147690e7944440f350b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/77b4973cc06aa050acd486480c24cc92b9ecb55e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/7b401757558339d65630026e90933e910451f41b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/87db8b7dc89dc6e358a21e906e9090913ec89f9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/a0c87ab5e9d00c9e6026be41caec382337217d92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/f523c01abeb2475a5d6aa02d68615a0751cdfe0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644867.89/warc/f867ebdeea6a46344c03b0ee3d235fe09aa55916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/0d9ff7bbb54ebdaf9f5e43e336b35791c3e78a04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/47741779bc28e4f1423bb9cdad12b8f2c7cec74e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/59b8a6b763e5dab4fc35dee62f99d01e44b1d663.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/600d282e543c8ceaba126f9c1276ca856fc8141c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/880953e24fc0947f37b7ec47b9ba64dde42ec364.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/d1ddb049415886a47b29f03548192df87cd4fb01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/d2304e51b6a938444dffc4c7223fc858e7fe9202.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644907.31/warc/e666c1477bb098af55ecb00dcba22a3e564d340b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/3f4e8d0a8b4a30c8a7aa960dfa2866cd4f09043f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/756c6c34709411187d433f12f1289c691fdd897a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/8e89570b67400b1b4e90e8e903a800407ef34f4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/904abb5a9d7ffa1f0260a965275a21480b47d73a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/bdeeacb3d6fdedec76c190f20e552cc340ead315.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/c1b62ede77f4898d4d1a172d09beab1466a465dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/dd5df98669ec2aa72004c4a7421c1dd1055daa5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644913.39/warc/ebc3fc99a8ffcd974fa4a789c11665155d46e951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/089685be4cc9cdc285c3627c8d5eb131e692a9f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/2b28aebfdfc46bc1fec3db95992be3fae688d514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/77e93b21857ba387aab23d7fca70cf43147e4496.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/80710d478c8abe24086e854d1bb920741deaf190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/92cdf504097953f189c00a54c0ebe2b7d47d2bfc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/c5688aa7ac1a0d57a39d436c4a77aa3fafc1b0e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/da0e9ce8fad1490c0872a9f878bc3c3fecb0dc39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224644915.48/warc/f22e055e2e89038bc5415efbf3c02f7b2402128f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/49532db4943b1526dd636dfff32c77c787593d78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/59a7813ea375a63bf8dc5aba3539cdc5d3d62061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/5be8e3b09e3d4466349aa26377a78315858f8f97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/6c8f33ca7ea3c17ff5bd15c306d10142d73ca09a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/8ab83e2ebe915f9f7359d11dbc3a5cab9ed25428.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/9ece29e6d65c6a7207e757602b534ba9eec0e2bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/eae85754255249f417606170ef388b7dcbf8eb16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645089.3/warc/ebb3c5348a03d4b3b39c3c3adf89685517e10d35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/11e7af1c224b97fa02fc72ab76b3209010136ed8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/6c26fdc8013c22d22ad7b0e790cc7a5f1c6e78e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/757442c67ab7d0e48236ee5a3b204cf8781859ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/7987623f4fe5f941fa623ecadc3fc9c1c597717d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/81f8309f40e73d9cc61192ff52bd5967415616f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/a0533aecf69d634f41da63a9e9b906b5d33dd53e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/ba33711382a29b73ce741d49b473c84d7025b721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645417.33/warc/e42c46b9867341e80b8ceb6f997935e1bddb1ac6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/2965d34ea6c9c45f3d890817574e9c2d679bbc7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/7c8deaae80ff0105a6fca4d0f9844db0b002ab0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/7d0a11701d4c408d227c320840f9ce45523021ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/83a3e0a93c49062eebc8bb6c3b82112734def37a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/8f5cf184d3093924718c7ea132e4e9d3df852827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/cf40cf456191114223fd60effeb68f7dd35282fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/e1826b29350753ecc0d0a5870abeb0e56df756b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645595.10/warc/fa6a4af7c73ea0b215736b5afbb780c3740534f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/180777742c14ad559898fe20f72760c68601fe02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/32961ce568cda44cfb34e1cff207c251d8c3ee18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/495d12b3a73f507d0443460c1a9c7ba7f5e907f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/4a0250adb51a3c13e70981cc2617749edb565948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/652ed0c5a8fefb0ad3aef9a4bd851211cde97872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/6d02830fdae9bf82033b0e95aaa90a29039ed577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/c1070b52d796681c7a5d9d81fe10994d823d1e4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224645810.57/warc/e7914843ddc048104c28e179df50e750c7ff0cf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/07d41a6cad8941aa29b0fc1565b781b306f58ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/28e826a6c69b58194ac50f26ae11a6c21ee89050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/5c869dc6c19bc38e9d4f4f3fc8455a0afe536a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/945cfb9ffc9c155594e9917017769cedaf1002c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/96c5726c4c2d1b7da6ce1ebda72d0695218d6d6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/9e400acfd30f6731ed7c849b9aa1c79df20a933e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/a7840437022c24b60115b90a7c7216f2a3d427f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646076.50/warc/e1a946165258fc842875dfc99b0e515edc39f8f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/2c14f992d00f6589eb393cb7c510dea6861a255e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/356b8610b0264960d0bac76d3579a41cb203778b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/91ae409c510778c275dd92ce80338524c39f6b24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/95773832057bb860fee76f713165ab2cbb3c15e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/babb407ec35ce97ab1eaa5a1a6a6856bac91d6f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/e1b1b8f3427b680fa536f6c1946c6e9441ca1342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/e8ee79517522a4a20775a41e05622bc0e994089e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646144.69/warc/f05751c8056c14c039b18f18628831dee8693adb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/0d3a7c9337caad1c0f97c33721a5b5d5d63b742a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/1e674ebc452cd57b166704bdeb4755c84e9c8f93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/2476a8c865fcd2b248ae800621948ba9a688537d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/4f702b9d9e950ce0ce7e8bf7a96ea2ab4a1eb9d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/639d65f8fa62ee937596f3ca662fc8b328ebf30f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/e73245831783afd8b331fbe9821aa1bad62ef697.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/f0d7d34b6754209b7d56393aa3fd79e2e4afe44a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646181.29/warc/ff82b55fac47539b0c3f3237dcc766508df8648d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/048820b5d11d6fcda33bcaeab603952977d377e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/1795fcf86f4e5ff9dad65065a86f41b799060296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/3148de32baca75a26d6c681d4e9f183277964756.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/6a8631008e9f811734e154bf0846dd241d2a7886.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/aa2e8d952449230236282a0968590a4382e781aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646257.46/warc/d5618e9fadec42624bd609a58ff253dc14dcb057.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/01283f9cdd33183864d3eea4a370550568421568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/4bf9c87212f7a87d36add49ea6f2ac6b3228617a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/5890787953b93d9a08e594519866b6f5c3346391.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/5effd9811260d4c3cd6d98ee6bacb3f015b25aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/9dd14b4aa1f67516d04ae113b7b0dba24c21a551.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/c304a077e2ef3a63b83ea54de8fab379d7911d62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/e50d46a12a9ce43331a6a64be82a6d0b8da5f4d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646350.59/warc/f0a05f923a78b8091e5335c208627b525587040a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/107089f83e969068fe227f8000d46d523a2deb03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/3006f7fc50aa3db5e46b3b0513c832224da33613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/3218c957347a625123a7656d656360c6c155facb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/597c78d0e73ab89efc9c46165f986851a9c8e048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/81456213ca885d702b2ed41362cb29c455730e19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/d978f1e0f2e8d2ecce5baafa16fe1c95eed47fd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/f0402fb349aed87fd250c8ad18f14ce95bb381a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646457.49/warc/f5a6547909a0fbc07007e6520d14672984831d2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/04ac2a937aee1ab31f0fb785f140839422545fe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/1da06fe5b891efc2c2cfbe3936a1b5bb01c2bbb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/360f035decfe8996248f6c22e8188f60c8a53c39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/36dfa708357589ca66c7e18801837bcc96625c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/4b4a8f054b1228ab234414fb634570017f3d1cc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/7a9a37a5c7ca22811a3cf82c239b99afc0a5ae72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/a35c52e34679280288ea7e2084d07390c01226f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646652.16/warc/beb6ebe5d1e7886f0b6c8fde41881ae732995e70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/0ad70885aaaee822f03759854023a0ef61070cb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/3670420e78dfe59cce0eb1231a869baaf3db1666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/623f8ad6c295eba3700e9f190188ced7c9f6f577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/80d0a24b2a8aa2f7d17ce06884d434b29043c8d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/8c61ad0e106d8d0bbfeabda77c806968b41b28c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/91e38461f823aaaa37c2d73a6f035acec7d4bddd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/c2d040140c290b8618ae671693a64740476d2706.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224646937.1/warc/def84ae011c0cc63aa1458359beb5330e928e4d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/2080401345f37c30d21287aa637d88ccb3c2595e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/2e480f2a53e41876df4b31b2fddd0456ecd2eab3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/6a4212db8b7f89a912fba3c8a466964b83e8341d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/8a221ef42dc82b41e8c5b64682f71c1b93201855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/9811e664206b39287510fe8fc125753bbfe83398.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/d370e318ee688609bc151fd0572ed9a4ee124ca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/df56d71fea70b9c2a900fdee498efc4a6516cd46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647409.17/warc/f046444f808e9630a18541bc01f8d822c1422121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/1a5e9e2e412206dee71273016217d94574d9840b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/3724bec1c7c71db809a34c6835a1c034aca769f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/3fd29f3cf0ee3cca6abf71e5aacf0b1fbfeb1b80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/600a4cc5e550a95bdf6438293b3494bd7410c649.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/61f6ed7678d57528e970ccec18af60c336674a92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/66cdc4ed73eed3f1384ff08d30212a21c46abf9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/980b1486bbb523e0b6aa840f96206e16151a4378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647459.8/warc/ba67777d031f37fa90dfa41b38b88177ff07dba1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/06b0b00967f81e1f3182a5f7dcc1e2c65fce3d56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/125d8752082e59253f7d36cd30362d179543819d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/31c35171c886e6de252df9c439d6f8fdd6608330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/49c2d3fd610597fa514bdf88ae37fb0da20ad395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/63d0836ddf34af4ec087c616665c103b9054fbd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/6a2d751a51d9e0c1729e5724837bbb463e545022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/8b0c182b2de7462771c870b2f0cb0c02506df90e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647525.11/warc/8f36ce51875283778d7d33d353cb0357312cd5b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/377a207e786d354e12f1bd92e67d2b438684bd7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/45312a0b405a790403515a0d1dccdb3fa0f9f39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/4f5f26fbf60631d70af3fa5959f314ad6bd6980c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/5323827c4e383c2eec6afa62e21eaf796aa34144.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/7a492df5060997428510b3d6999882240a2de9b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/7e10442097c90f094042cb1beef529c9e75ee15d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/ad9fecbdd0b21547b7628a5f194d8395dabf6027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647614.56/warc/b35868607311012d35a29c183e3f0505ea493198.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/38fd4fb7777d9e494608b9bb7605d6888e6a27ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/6a97a855e2f2d709a0a5db0f861bb79294616813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/70eee31e43efe40c76ee50e378e8f1f2562486ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/73a3d4a0e4dbc9c0b0932a36e6b6325b519f54a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/822d1cc349ebf0d968e3add7bb4299f861f1785c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/9f27fbf8e1f486546398ba424e9b08c48d2b4f09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/ab979d9a883c8e86b3bc0a55765a76d82462dccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647639.37/warc/f54da55fbc8ef2a4ed3f2fbd808c50d437ad7a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/125770ab1ca285ed6d021a67b5c284fc722f1ee5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/7a8b45effe240badda85a8c3ccef7471a2ec4e30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/7f1d61e6d54526cf82c42582d6488b9d690da773.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/8b9274b83169d8e8a588c4c97332e235159d8050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/8d5873481ba30b4ab614b00e03e0ebfd34326e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/d682063897c921ba66c2a2817201f0ca9a7f2490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/f95513fb0ebea571d12830f4c6abbb6aba24e884.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647810.28/warc/fd512184576d021c25014b5aba00f6f1c961845e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/061abc07c703a8bdcdad3422972dea67cd334d8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/1ecb8b7d4d5ac4e80199c8bbbcab815fea120380.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/21e7316a4c2047ce303eea5ab0a10568fb47d9e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/3176c712929dedcf9f76758ca4c7ce9140f87d2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/93056de2e992737f2a75495df2986bc6d979336c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/b5d7fe9423495fec14b3b36ef21443d330149080.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/f0da6d2c16f4c8544d9d63b20d1271209d109368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224647895.20/warc/f86e3f2738b2d954d0fdec4efc2f3f79f1e617b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/6636cc26c88f60447fddb3b8d3205551af4905d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/7e9acd4dee2ea9dadc816e8ec9ebd90c6a2df83b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/80e5be7d6c364228ee11086c832f9066d4ad1229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/b184402f367c2c1941c0af866d573d3171968cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/cb68f152a6fc1e58a3db793933c3804778233f0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/ede8cdfab0d9aa77a05cf8470ea254d8b148c13b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/ee2464df770cb4490cf16e582582d3ccb4e7826c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648000.54/warc/ee45367cc8022ec2c4c379a39df75b3743b732f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/0eb28f5ab7f633a7f02d7b4e547302a084224da7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/0f4cf219b5a5d059176867a19b5da1ab7011b6d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/4575741ac07f03e82a10f69f672c01d42419a9be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/7a5774c282feb4bf83c864e345838f491ed71fd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/7f570c251a93d3d7a71b1b3a126fe374e2168c35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/bd35c1bc6e01efab567ca59a920c1a177bd36da9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/ce6a0e6d823f78265f94f41729a5a5d2707d7b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648209.30/warc/e8d07e87ab2d3e4e1fc362fd81b40438418faa16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/236e0d1ff9abaafa99ef69f14a44ce996bf7aed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/27c1e8ff0c268f36b0faa36e9ea53235e289a78f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/3b56e779838d394a29be6694ede4b712a8581269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/61e024742f7ae56890d70f3ed3f27bf5dff32a36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/9ab7bc31d2aba2c4cc9d61ed0d3e3da9404cf4a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/cbe193aeb4605f792463796664edb0d95278253e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/e398ef5e5e5112f5c66ffa8e660ad9ebcefccb48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648245.63/warc/e5a1884f59b3d62910d7b88202bf211cce256558.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/1f23886806f01900cfb6f4765a807b898a6efff6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/2bfd4b4210beed0e8f513ca834bf1fb1911b1e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/37c167eca93192b1c2c60acf31acd2d366e367a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/4aa6cc0b8557be2182bacc861f6b07cc7d6a71d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/5780ca59c5d1bdb8a41ddca3f5d72b620a4627cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/a65ec8750866a283f7fc8d9624d6052b80368038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/b22c3555a0b980adeb0145497888c081b4dee37e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648322.84/warc/b77fc342a7a0f9dec65cc2b46c25d674b3e41d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/56bd7710701b95e34dc05fe6602c1576ec869524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/6338f9cf4779ad5725bde7d89054309a2da186e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/812288f93341ec4f6fc2f75d8fe18e6d565732d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/a2e197467073ec583807a205279e3c39d2cfe553.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/ae91bcced83aa9b9450daffa904552ebac27d8f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/b1b3b8e511e296f57f2b4c2031ba63a69bfe1c36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/c282d4954dfc441822e94abfaf7ade4b2c9351b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648465.70/warc/d6b02b205af3bbf0168146a2b9c5f858ff8338b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/10ee6363e5b71389289781b018004388933c7b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/210d3735b9e974008c4cbdfd47bc2d8d272c94de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/293635937caeec36f0598d35079abf43d9062de3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/527b80dbcdf1cbc5f8ce29d40b020a277f42cbd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/a86399c9c8983407a7e19d4ee39b20f22ac63d2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/c6614ccaeb07156231e55491363440f979ce6278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/cc0ffa2edf1dca36c0344444063149957db7fe98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648635.78/warc/ffbf23158fb5b90f61330af7fbda9133ec4429bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/03d07a3bf989b40fa34dd00b00548505e29ce8df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/0cb1c0d2b57b7aea7211fffdb02f4c0cfff4d2c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/1067093166a18e5db1da10ec59c902867c321f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/50df87e45523f829fb1b7d9398278f8413a932ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/7612cc53326106c2d2553436931dfe3898130bb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/89127ef777818e93c86ebb4ffc0756e425ecf0be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/b3934f5e04480559d69ff22cde56edd49219da1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648695.4/warc/b7fa7c39484b1cf88d98a88df1a86f6c15c062b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/3923e62b1b411a8bfcc238406e58c159152bc829.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/770ef47f6bb4eb4057cef82e80f907454c3f1aa7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/80ed39cbea6ca6a60acb5454b83feb4c580e7bf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/81273421108201205a19b39239cea00ab6a6fc0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/a04f5a3579c9ed015f570fe6f25bf0d75ce792f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/a0dd44ff4648d667228e0b4089277a38ea56aef3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/ab3ed02c4772d7fcf9b1c9ebe01a4013c3bef54c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648850.88/warc/de9c5e0cfad93c62affb7729b44e6664bf2e3758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/256b8260c23721adcb050d664cfb6e5f99568091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/701bdef43527ce271c4d3f5722d9f2ff18dafd22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/757abd83d3c896fa47fbc5b0e80b696d20535341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/c3a36c594a3a8e7de0d87c3344404b96e9fc7038.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/c78c19e8c47ac09f05976d531fb1473c85fbc463.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/d9109af4820f63e61ea63804d8f08e1fde3d141b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/db401786cec040e1a8db353032c0312b8638c703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648858.14/warc/e227c1ff2151143c7740256c401ef6c7912eec99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/30b07c36d037147a090dbb2e434d43ecf8cff73e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/30bef0737edd7464e28e0b0da39212eb44979fd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/53a2104eaf9463230313534862218da956788a0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/9dc354b4238ff039960a8270cd1ead2803547e0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/acc7eccc94a136b589161d7219b68c80f5b3abfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/d97b5b03ed1a59c7c93c860cf323f4712bcf00e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/dfca762fcaf915a548664d8aa0062d4e4255b0df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224648911.0/warc/f9936a720e48ee4052901e166bfec5badb86c372.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/1ef44f063caa709e2c277ac52b1a628d6d34cbfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/2322d4275199a30c4c8477a1197c4eadd3deba30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/26240e7c6ad033df881e336cd09f3c1028f7f732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/3e36112d8ae29c662b1e443027401c04888eea45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/6c79ff174de81a4f1bcd731d98738a5c5794c118.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/84c7f9aea8f6decb6d08323b24bc8c02bb89563a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/bcd0c272324f61e61f5c92de397aa53e6f31f12a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649105.40/warc/da857bfd53346599b979b149da404e30b8ab4d1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/0414ec38f7a343b7ce8077133b33fd2177ce706b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/0d6ef0ea4b95fddb34e4057a39f907a66c26534f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/63e70382a2408597ccfc28b7ac8d0cde79e49a13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/6ebb1b6d0d43d87fa041bf83c69d79f7fdf8be64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/98eb1343b5fc6c72f272cd53a8ad800e161017e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/a44f5bf04fa436303961463168e58d7ff3dc5827.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/bfd0e54250fc30d6b0bfbc976ce4cd2603a2c23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649177.24/warc/f14f5657ce8a2a6f7ae7c13a72bf622a9435450b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/1ceb766f74f7fa50d27af8d0a33e2752721c4db9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/25dff178ed97bfdf3d627188dedf4d700d8c652a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/3767d6b3d31bac0e9befa57f52532bad5fa5fe2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/7fba3fa7da82014116a9e9301a0fcc67b43a7e02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/b8a6e68deb9d852a1ac6e6dca6ddc2cbeb7cc696.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/bbc1e221ebe2fd9f30439866700daf9b1d872d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/ff220079af2aa6f0c13d9c2e9a5c7ca73131c8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649193.79/warc/ff9b5f9f6baad9fdacfe6e75ce626e2d4045c07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/3e24c6ee2faeba58eaaf9c1ab673340acfc7c5da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/3eb118ea6e72fa6ee0d84d71fc2f4c81de413350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/4e9b62b2a486345cadc864c75566476fe0a4e005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/66743d0e8f03d4f637a85157fc795af6f7b6cbcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/aa030da5e2220d63fb91a4a79afee61f4bdc4fa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/c403a22b5f9ff962ae03bb7663a89c3c3b0e7fc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/e2a2dbb18317190dbc157975274c831a93ad3957.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649293.44/warc/f7bd679213e094b5c7441a2ff1fa16ad17305557.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/5522b75b9e8fd44f3f79d251db4bc97ac2c5e853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/9001c5a972c67a1d487ee30347fc1311fa803c13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/94693aa6da31e04ec3872d9ccbbb3857483239da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/bda2980a35e0d18b227d80e596da5ce94315b1c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/c07c1ced00750f258eb13dcbcb629346b13c8cf5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/c17a4d09bbf470eaebe72679deda19886a5b8f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/f4298fea9b165f3843f313fae6f3c71c7e50e47b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649302.35/warc/fd3020764fa507a89db82faf2e16fc6e558f0929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/0364c291cb3bdb5854ef23a165fd7fba3cfd03e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/17dda878026352275f88b6396b80be2a2a8269fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/256d8113e8087b4e4d4794fa0c8385964d05bf2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/27e9bb161e3dac172ffe1cfbc9b1cff43decb602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/626ae4a4b9a8981dc8c1680e782b00b2b9d73ae6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/8326601b5bcf45f5190c9a2b5eb7cc2ef242f833.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/90b63cf7505172887d7504676344e9f47f78fae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649343.34/warc/a51c98786c681a5794628f8ebfce9d3618b2fc54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/08927d69daf6766db9fa17624aa2b207df1c66e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/251b2d9c97ac370c445ef42c8f264d3619afd58c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/4185a92c5086d4c2db56c1ebe07cbe7f04f21279.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/781c76f9be630a9966266d6f4f932b0490776542.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/a67e51781039de62de39cca53f7b175cd66f5350.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/a6feb5a945015a512a6be3677be0ac7d370f349c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/c3a33ae8f678b5bd9b9300481056cacbe5327783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649348.41/warc/e8c18dc022b2a3cae7efe66ce9705db7d4039daa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/149331b8810b7ad9cb7f91469c16a88a7e4ded0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/185fa11a6dab6c82bb6bb692dc367a441d3ebe06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/43faa262adacb1c5b162ff19087572a422b0c1b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/78c167c68ba373d3ad2038a7d64fc2729ca76a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/7d948edc1b4046f6a96f1c9428e87d856fee5a1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/88ffeab2f2b6779ba0655f356e49fd8838e22a18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/9c3a13c6bddc547ff7c7c2d1f8d32ac728c87980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649439.65/warc/c0c47be6c19e4b5ee4379e2c3c4d1ff772815b35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/2886d98ff67ae7b94cbf9086e3621b71da48dacf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/29579a2eaeab2757bbcca8cf6b875e8143a3a56d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/312e69815c2a6183e0c8c33d37d3a79928a3af1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/3a94bc3fa4ca24fa819fc516f0833038b393cd2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/3d965cea902c325c1d056c71e7b30c321e4b44e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/6d49b91907df9349f50d4807d9ec2d1be075fb44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/aef7e65e7a31c29f60892f3a6370db141a614ca5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649518.12/warc/b3dbaf755f4f6ff2672d201405b602cb02957274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/03ff429783d8b08e6594cd79fbd734eee1f1a71d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/0b8aa48c88b417baabfc7645f77ff72ef6fc9b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/30c72e436c5cf38447dcdd38c90f719bb4c0f9b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/69772264a7382131d3d1a6ec2eee1fa9c99e151f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/ad4e8aa551b562aa6ceb79b9df6d8e56f0904a60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/c6dc8e6e1a41907050fe17226daac7e69ef7f0b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/c96d18abe71fabbb17c0ee6b35017b565bfa1cb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649741.26/warc/e3471d4cccaaede2b515576f3d9053f68ef25934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/069ccd027a2afc613308bc078ab558087a06f8f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/0c31cce1afa6f2ad5fead84966daca98c5b18740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/168b5744a6c559a65cc7d24a7e80ed50e8905aa4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/1ab5679ca033466c32abded19f359f5ebe72fae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/44ea4c79d27ae7a781544e347331e076d2aa1cae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/4c5c16b58dd11bcf511486558a2a6ee2c445f060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/6ccbdc8721919978f062223b616fcfdaabeda65e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224649986.95/warc/a22fcaead968ca38b28e577c5e0246818e23b5ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/3684b65cab2939801aa7155b283e8396334df048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/4ab428a9ed4425dc946a582b2f8632fd0c7a9cab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/4da71768764fe693e316b28d47e831ec08c3a550.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/576c788d66407a6e12997c7e833eaaf175dae6e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/59104b6c99e15786ecfd30ae242c52995462fc45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/95f8c888e14429953807364b67fba1b7675c4189.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/c02a44094ac9d416b3574d441af1e4cee0d0cf31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650201.19/warc/f23b6ba5d58e344f272339558fefca008308b7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/104a9848588b3caa8f25df0409831a6a3340f163.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/1a84ee62218d442d922dd5ed220e93dc8da6567d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/3a2b0dbf2d47f89e651dd6bd8a01151e1839b509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/4efe11936ce8e773a7b90841063ab133abea60a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/6431584fe6c76f7ad0120ee5068001a5f43f6445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/65347457fcb621199ec01110a871abfbe1b8dfca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/84eda4509b62708ef34d146275168c90b70bf9fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650264.9/warc/ec3e1b9bedaad0b081d4b1a1ed2a6d333aa83f5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/272a50667b4499f1d247782449795c92f73387cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/3d0f5cbf614ad115f8421604a874e8adaa724ac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/4d08d37d2b34dbbdb40569b673029688579e06a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/70e83b07621c2bc33c12c07815df258dea348a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/832d31cce507e253d8bcdc6405ea1e9a3b0d396a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/9eba48c9f07fd7eb677954d19849160a9f8267b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/9f165b90f38fb66aa069497c8437fb8a98350555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650409.64/warc/e1ef7015788350904b544741d664cb35e796ba94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/0ba332dfd636fa3a782b11b4fd53f91e46dc1f88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/263e6b04c4c6f3f10583fdf5f397cade28bf9a4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/3655524757baba10095032af9813f5b20d315bb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/46df8081dfbf72450abcc3036d9ba4a00fc7551b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/698a58067e723059041a168d4854a004506d01b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/8915eaf0b75bc43e650028995c3004d32eeb0805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/a18f5d707c52c6a1c8b38cd97fd06c70ac6bdca9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224650620.66/warc/ab221f731c27d8caf59a8d720e3221d1fec29e38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/37ac23313b791134590f3603fca9e4d3a98b366c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/5ad21d3a3e0db1d9308d61628e8b956c6e4cd4f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/799c83aa5d13b7a6afc027cf59a175f107956bd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/8bfb31458334e4f73ea505f1223340367e55811c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/924ab4c709179249df75caf8b52bf778d92ce984.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/9a3c7dab659a56ebd14cfc478aff975f32a29358.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/b3bb5758ab86117e9d7b1bc6c1ea3b1b417f8cfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651325.38/warc/cce88a1f33ef06f6fda80cdfcbda1945cd1f5c19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/1b3761aa000936341fc534399691453fcc9ec291.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/2e9ddec7ff71f63079cc80ed889337eb5e9c9dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/44264802e14d8e2b2f931d544d4b28b8973a4604.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/5fcc74fee93e5222ed35d6de5b634d4341988dc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/69382b8d0876211a31682a2a7c1e8fac19746e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/a84f96296cc27c29f9648a9315517d7f647e5c1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/bfe530556b8de425938e724e0f1fb7fc1a1b5a07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224651815.80/warc/d9c5b21bfb34a85a034c96432de8735be463307c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/1d9df5cb198b771ce190fc9260ee691ff76bde46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/31388c46e17d72ea605ddc97975bfd77b71072ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/372db6e5720ba6b335a7acb81e4a036782286d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/76bfe966e16a77ab03a93c95909b912b99df324f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/984513d5d40672394776656469a1c1132da8ba35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/a2089794d967d2b049927129b94aa2976b97255d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/c9ae721d58185cefad969bc602aa55ced6e943a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652116.60/warc/ce2449baf4ad1abc030ace340ed81dfcc3bdb87f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/40844fb1c86981b8067baa3d5f11be0caba7f74c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/56c9b9a225067349f6c901adb5007db182d37272.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/60bd119a5cec7c70533e9715db7fbf1707bc2882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/7893d8f3bd3078e245c25342d41fe0cd8d550e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/a5b105967fb0820a72503ca3e7610ada4d9cf9ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/b2412b24a0e57b96f1fcf2e2e6bf96ce5fb909b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/cd6017eeb9de1a0a18ba573cd0608052e27d15eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652149.61/warc/fa69d2d59da69a4623fdada2e45307c8ad6c5fca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/1810f19dd687f4c2088a3a04bcc8508a1e11ffc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/198946599a0a0d5e18a4caf7f696f1c031291b5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/4169e4834ba69007a1195fec7af07fb5960784ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/64a52a09f05829caa72070f8ac3cf0873724d732.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/723bada756a7decd6505afacc792337c5eb00b11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/9dd4a198f22fcc518b0f402a9ca089190f38c727.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/b9b27c3e3b7e32e12fe06aaeac30283569b0101f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652161.52/warc/cc5a964638f8768ede024d9b4420fa61ea8cd531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/0dfc26b706f4368b1601aca78e13a4f93f5cd531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/24ab9b30831911cbdcb7bab2b94fa7c619c871bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/2f2d09784239291e676f2814eb047226a0356ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/35b1ad73ac8e595e1064f24171dd32a7595700bc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/4235fd15f48a75048a6c9da202bea2472db548ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/6fdf6162e77e6be110572f4713bcc7bdcb49ad05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/7f2882afb91cfcf3e8bb818333cd8c722f318beb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652184.68/warc/99d05de8ac3bb1f1d63d30d40f22a19746866be8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/247671ca992590b2ed77e1ef9ca9b46b1612262e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/29e28f4a5626b9d042d8f18b4fb571c871651388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/465493885ca73887c1c3578f4a1a55bac66ad68c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/4dcd70aa7d25e5eac244db817560a17faedfdaab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/8e006710d1066388e323b169fd1269ea5ef3e2c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/a0e1f0358831287ee0742caf27da77dee80aa714.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/cf055dfc3cf48db8eb9c70c519adfc60b4b99c59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652207.81/warc/ef47187b9a20f60d03f2e27a6d53b6e64edf2659.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/0ab91cc5773b8e89f5c3af9076897340abbc2396.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/213c6aae521dad3745b40daa4abb26f01e0a9a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/381fd579a472b14322e32482cbe83f1694c5cb6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/3e0b29ae07a1f9d6e435b2172a6ce1f4acd1ca6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/82dd963a4bc86eb1221ab30c2fb5ede412690102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/9f745dffe77576cedbf9b6e37b2a11a573ac6474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/bedb35bd575eae5a7991d1572300eb935618589a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652235.2/warc/cb75654271539d08460a195031d5bc24805701e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/1efecb24023406662cb275ca59e9fd7dc023b67d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/20c2cfd6fbd6ce9de7ffabf2da8156a631566075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/3013a86eb64e34dd4ec89fd22dbf14c527c144dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/3ce894a2d8ff28ddd7d53b7b6bb06e6969899931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/41d200ed14cafc9f712009e96f900a22a20e9e8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/7f2e4d380bedcaf91d83cbc69720c473e77c2e2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/dbbc687de30504f118e1c989d5c0b04065f4b168.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652494.25/warc/dc90aa6aca954e2da08fa7a275e2fb46b79b3c34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/1e0913b4506be520dbe38eefb8e1fe1642a47973.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/22e959e864e4d5fecc371060d5b75997b928d0f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/3c4b9fc09e17d8a90a8cc779ddc6208f0ccb5357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/5845ebce6bebc274ff832de53bba83d98e12a9fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/76b4820d27ee65fd5aeb58a855aa1ce1042e1336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/884a1f370690d4396fec6c2e81db6b77e90c390a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/8b17807856cb23251908467b0caa55e2ed37112d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652569.73/warc/c8585c2e44840ff70ed9e62e08d2151c19803db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/1010a4491ebc4c8a101926a7e8e9319eee266508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/49ebe7e0a17321b056d93ec4eadb0eb4123da178.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/7999f76a1c45f8c6aa69d0b412b297e8328385bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/846be8ea3257c2fb8a3b62cdc7a83acb7c685916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/887eb317973048fe0215d94edc289dd917f4c724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/b1e5f75612f1207ec072fe47e0af324adc7ed0c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/b26318054598402337db7c059d4a66895ef76d07.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224652959.43/warc/e0ee18b669d39909655148733a21403a9563bdcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/15cf26cce6fcbf1810c5c686817b119f63c76549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/3cbb4768eb72a7f916e153b502fc35d3c34d9505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/583f1363abb8b3d1b17accca19687688886d9699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/8f8a30bf9e089f32e2366ec71c7727c86db4d9f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/9b37b564635d6c6d0ad1878d1c1dec9191957506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/c504338469ecfc20fca9d2efa2612f75a12a4c68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/dd48015696019387b437e4fe67a189deac8d0c88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653071.58/warc/eaf2461883c65877ee9f6d54dc4784eb07c81254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/108fcea898d410edf0f72353b04e768d889e1f74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/41b5c67a70d320ef85975ad5a85daf3682e87b2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/4a89bc788d83ffeabe36979760ca0b0f0c99df0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/5363f7e1fc2d08b694f76fad7126177df2499ff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/66ca6bfcbbdac9c6b865689650a5e1f71eb3ced4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/8ccd13fbabb38010e4f7dd65433822e61618a238.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/edbaffdcdfc8c2436ee4814302cb8415eee48947.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653183.5/warc/fba2890325db089509065f9bee81f7083681f442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/1ad2fa23d9afd9122b93dd73bd0a06034165911c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/3c3c8a4b845368b26f1426066fa14d850432a510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/8c75b5b475fa77b79781e4750eabd86260bf08c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/978f6422646f0e062f881d4f29626c34e0a19e4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/ae916fdb5d82928fc728a6bd7aff8059937de37f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/c7b2018dc33c7b45b4519d52da10cfa6a99568b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/cdf5931e9be345915227b9d3c0111bb09eb837a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653501.53/warc/e91629e787b7baa8aa41991d8e017e959e95edf7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/06603d650bfb6265ceb37144c085776d544bf65f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/2215a2f8f69e0ea0d9443dc1e74ac856865b0241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/413aee51dac66fbeb6e5b35f8d4e3c68dd55e434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/5c4870e50b1021b57949d1f476918bd43de3a1d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/81d278e84edcaa076320620434390923b9f8fd5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/bd11fd46654b0516e7c0fe3e053399926d3774b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/bde4e306eecb1fa188aaee43705f4c91eb9ec666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653608.76/warc/e4f76cd56601847d248fe4579147ac6778aa4498.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/14144c6efaa2ca425592ce248ceb0b9d720ae4fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/1d568d50bc6b01720f33915fecd74dbd886da2e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/2df1ef05ac34c4ff9deb0954401cea911bad10be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/493fbaa2581ef774e4213b00451c3dba4b087b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/a7c479ec6eabaa637439a8f6c411a7a6b3ba4541.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/ab8193f0413dbcf106f9f7595f7803da8d3306df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/cba4de8e915cd6e53c8fd85a1170cc68d1f12e13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653631.71/warc/fe23bce05d74e7254fa1defec7b77c615faa7c86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/1dbc7cfca6a4ac8475e85f6d413de3fa8dda0de4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/63f712c0e6df7fc39d8d146661cf723c004011c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/6fc8e133b8d7d7bfaa0307732dfab6b53ead7e49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/7c26b9faa1b374e7e8735982f49e9be92afe51db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/82b029a854411ca553a30d0f02ee4e659f6a0419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/cd6846c324646d0ddc79057aeb4ca8b73722359b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/cda53b816a6cc7b34a2f6cf004dee01345e03940.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653764.55/warc/db363d2df100362f34a3381b281b7f8fd0620e64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/1c8793ca8d7f49e985e5715f38a764dc5a34ed94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/2d9071925eb81f2f8cc1822ba758298af9f3d9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/5982c3ccfac7d76e5ee770ff5408a54aa4a2701a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/bbc9aeebb2f9b064e58090b9b07939a14a77deb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/db8148126036b1de468cd7d7387cf4cb3114619c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/e32c04dbc0fe4f5fc63478d40d55701b8447d871.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/ea02870ca3dd04b689385ee6c533a9dfe89ea1b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224653930.47/warc/f5d11cb419c22e17497229542e64de4dbe8018c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/41ef6c07baf5f168697535dfc7b310b2c7ce1519.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/464d9fff2ea290713d80a4c9b22ad54c275280da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/49343b1034e9d9d1f696bfa4b609e8012f629f1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/52cc51665dc9bf5380556a3287d44031c6fab2b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/631fb31c3be53488a68d8fece6b3a64e28ebbbb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/6d7d93870b3b778e7ea20a4684ac51d760ff2319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/b7bb6662a99020bfa8201cb9109677d41c2578fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654012.67/warc/d0be18d729e944543054009c9a4021619262d52e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/0b4d5a0488c5d0f1d52ffec1c3099cde4312c1d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/1466942cc767864e09847970dd465ef345976a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/547fc2ab869e0061fb3448684dc11f5861300653.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/57b1f6347bf5844c6d49a06e9541d870d4e5567b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/abb10b946bd3376df1886427da78234bd72d97a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/c12b8ce83d295e7428be35f658cc9f36eb61606a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/f2efbbad6c5d30244d6eccceabf53e8ac68cd4bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654016.91/warc/fc644149b20d499907ef3bd79b87ac8d7b2b5dfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/02cddf817e9ce22abc32fab2707d0b6c21a13241.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/1254fcb1a546f4b0146bc0be342d2b9fd4f1cee0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/1dfcbd2bf28201c979eff8ba6fa3f7bb169af382.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/5ce186d4d7a4ad65a2c0dfa5f8795f9000c07ecf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/8c0141eef5744107ea416dc625c2ac8bbe7a5707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/9762065fcb66bd037af5a35017a0709d342d102a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/a6a342b87b49e28a5fcbd0a8690c030253305936.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654031.92/warc/deb0720a8c538608fbb4eb1a4826571ecb230b8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/115f5cdc1f4585c2616a6196dfe813a27410423c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/234545ed9aa3f4639d03958e1314ed60634d4672.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/29a8f47b4530ad1243c89b9d96bb02d759909f57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/2cfc44e31f0aeb593faf3e6a142a9c5aef76d039.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/5fb79ab22e03f45500b324899ac80d79d7b70f3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/7ff607185062a8e1241f537bfe32c141223bb219.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/97a9838ff8dc153ca33edc7b33627e629df3a552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654097.42/warc/9b331a6ae48b68a7faf132dd1710b221049a1b05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/038f01d65e6c750dea5ea4e239d90489e3ff4744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/2789acfc3b0544d80411f2fbac5947a805e65064.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/725e4b04daa49ae29a58bbeb05e353a2e573ebe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/986bc19ed2be6b42b64fb4be5958821c075dbe4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/b3b160a8dce35799e9bd67893d75c0ca9c3eb755.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/bb28f8255a3da6593156d06feaa9a3ff6bf1599c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/ee2408a73111898b1165aa340993ec46ff82191d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654606.93/warc/f92bec7045af46ba542dada24382ab8f7f420d99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/05fc92115d58dc3f44443b5304dd82df74a84ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/170e7336f85bd0a064cbbfa861ce9f31e566d3fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/37e3daaccc3360f9e7f75a17bfadf405143e1b7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/67842df0c4d4b83be3f31fee22084fe7d3e83c97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/7982b14b364f6ac822acb3650c0cc91f285ec3d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/7cf49e1804cb1290eb52ddf2423357ba67b76375.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/8aa1b163bdcb8cb8687b372fefd2a9849c4a3df9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224654871.97/warc/928c6df47856ed2aa680340f3c058e0794b4dd2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/00285ebf323f70f6de376eddb4ec7f47c8f1124d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/124e5f8526d0e81a3893dd3a8c42a04fe2b0b0c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/16f1257ace776b707c43ae8a992fd976c08530c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/29c2faa2b1ced87bed504252bbc8864a01be244e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/581d021fa0702caa898bdaa63c02f6aa02ed7f8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/927cf9a8f85ec3e128c13f1af57214a30e869361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/d45c31153c90bb7fb5de7c433fb7cac9a425d48b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655027.51/warc/dd9bfb619435c06e4571acf7b344ad82e45a357f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/80d04a76c34817084a96e47ff5da08d02113d833.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/9fc0bd83501d6489d23e2e049568f9ef19d305e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/ab785657a3a56c7131601531b82840492b827389.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/b2096b47d437548a08eb7c91055c2e66bcd6d602.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/c175bcc63f1deacf685b74dfb364c685862abadf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/c36d288ebf0c4c267c9314da958571fd22f54ff3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/fb1a078eff2795603760dc6007bb2bb875b6a260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655092.36/warc/ff30197d01f57d2fe93972732c8861aa0ceb1228.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/10bb542c8ee827925eaccfd8efde42b940235a37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/11f40b46d53df901cdce57ea60f4bba94ad9af79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/3ab1bd85d3b174412207aee856719b4da4c43ba2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/865fc92b24854e27eddc3fb467f4229c962b7b85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/8909c309a4d7c15e5f5da1ba0ddbc62e0ae09326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/8fff590cfe2809d9126498b0151ec0c129848c18.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/aaf38ddb70b1cea5760ad271c826b28118237ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655143.72/warc/de9b38bbfa98f597df7105766baa46a153020b70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/3a9043022f2305041a6f8ea18efdfdd1e00fc13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/4dbcf9df2ae68981f1329852e42b13a60eb6c3e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/7837e6d1c1a56a9bab617c0fd67e337f4b3be4b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/97dcf4fda5efe33d33dd5d4fa47a4574ed8c0d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/a8c82faa258a465340a5fa0c946300353430a88f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/d3ec631020f83c801f5f9c6feb5f33dbfe4e93d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/d8ac4800fbde39da9b59e0c4c4ccf9c9ef33482f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655244.74/warc/e54cf32577d977e57a9ea27480b2941974e7eed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/22cc787a99d22aa2fa994e07fa47634eabc1b9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/5f114db148f64c39da4a1d6ed0b45285943dddeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/988d9af74524acbda7febfb8344be8819f9b2a88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/9b3b0fd2a7f0b35d190a11ec78f43b44ab6c5418.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/a84b0156f4d057afbce1747639b0ec033211553a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/c2ab850c28b89f9700fd77bb9a340ef69233deed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/d7c307ba81a7212a15399732e409d92eded39657.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655247.75/warc/fb1442512aeeb578c9aeca366911b2ece2211f08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/3c57d65dff7f802fb4d94f77e6cd3fee1b636fd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/4d581b906cfdae390318db57f47a74c0c06e97a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/4eccdc0718ced071177b4931597dec11896190c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/52b2ea22ac1db7d6b377de1e247457e80bd43128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/743512b0649ace21e8cfa30c2c92d525a822c012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/da4c5d62610e4d7dd57fa107d71addffde40f4f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/e6f7cd10b5a21f24bf68708b22e217babba344e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224655446.86/warc/fadc7d47fbc1deb3b606c4eb004b8efd4bf63a1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/05ede0d2e67f2b647fbcdccafc66ccb9bdd138e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/32c5d2e3df6bcc1bf3a0fb62a34dc6cb4e1b0252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/4f963896902dc044c687299faf279c8c283051f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/5317a8e73969b0dd6cd9ca572318ab2fa1d16851.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/572e53f5031051eee44a0c968502c6e13dfaf246.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/5a41ccb449c38b43ed2be8428a4c42d3b743614c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/9a1337d662eb3c16a80b78b5d5a7a3cf0dd88112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656675.90/warc/d7c0406f357efa7004dec58917e1323f704724df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/365f88d5b570cb0dede2c11177e63aaccfb62ae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/4a5665aeefc719939d76f6c5b521efc7339cdfb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/940a9807cafbc9b1c765db768229c62a5b35388c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/af549c09befdeb896c5339641292ff4c73223562.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/b419f5256c20074f66502829ae08bc464647baa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/b988e8d86ed30a07d706528f159ce9dbaea70167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/bcca91b4083ecf310270739671b149918c6160c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656737.96/warc/e9e8d927d35eb101e7ca9723b0392b57deced814.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/2188fca2a9816fa3cdfb83ad68f872440e4840f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/5e4f2351f073b14e389f22cf653ace4b05cc7fb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/5e9a775a99acd708ff2ba655ec4efc53dfb2b25c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/98df3c8c9a0ebdd81787f9addc45e01d8b84f64e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/9abaf941a357c085e9d1d1bc85b5e81ff60bc7c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/c673b7a811c6885e5ff1a463e0d5dcdc9bf691ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/cdea08d3b703fe010064ce8f81e346adb81505e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656788.77/warc/e814868f782c38bcc3e19e999ee850488d0f4fea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/16b88a11c4a1e12b22812838a05f0fb53e9f6759.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/23ec1e3d768c5e5598a752f09081f084d655467e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/31563d1952cc86973bff2a0ed12f48577f6cc3e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/7a8a474ff2c516bcfa05edfe2fa59b4c5df130ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/7ab7a4db36d6b5001cbb6091ebff6e46af14d2e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/be5a41453dffb44601bc6300a50abe2fa3111776.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/d174dcd4f2936c89ecda83770c38cd0f421ecef2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656833.99/warc/dd07bca84203dc19a178cf0e627ecd2f9be5265a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/09f04ead526c64f561d1398a962fd1e6b41fb872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/118e6cc3bcb70546cc85fdbd301ed6e0d137ef55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/301b171a66b4b377150e4433ffebc38109e1f793.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/754827189f684931774b509c1a70c47826ebc21b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/807321147ca2d6c75166269ef5828d6a5af75d82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/882a398e69b312b741742f8a49451bdb5b1710fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/9c201c7c9c3ee62b691af5d2073512c2591f507d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656869.87/warc/f3155c7edbd44742e412362d90eeb8e9c47f9af0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/364cdfea02105bfb485b0d49b7147810ea7aafa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/680c86e47e68e1f375cabcbd9c6a2085e71eefac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/6875c290bd5d5aafe54330a7a31d9de0cfa86ca3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/af90dcb8e14b9c87c9b0b361f9923a6026dcd193.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/b4f16c625a3f2e7703830438e559b965adfec86f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/d94e592edc6b782b73c62373564876ab3644594e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/f0ab1b22a7c2ab161ac64cdb686a99ca151e0fad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224656963.83/warc/faf8a8d383cfa1d975147146762522eb3eb20d89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/0e1c31a0031a2da2753e286e682a4e6c639f45cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/1cdc691ed257d9dea4b73670d746c057b710f6c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/432030e839d2d5075300a5a3b22ac3c0865425f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/535bb3e69a8417103095301902dfdc1b145f29d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/57ccd8ef37af81de9e09cd52bb4491805d29dcd3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/7695c493f8a0b62addf63645b22866c98b1c9daa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/c29fc6aee18026cb3e4469ae01f7160549f05ab4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657144.94/warc/ee146ce22801948eaee6fb47794a8227116d8fe8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/055ad88e3eec19f4b38dfae9807d78f198e3e9d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/ae0edf3deecf74f47908c0f66f03ee304afacd99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/b5d5f94cf7ecca995f678143d5936f6fdb4eeaeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/b8a6e55e6572db0d955860a908ebfe25224ac41b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/d1969515c6941043d00e703669b3442e41c5b318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/f295c85c343d9e4be4235ea40e2ea6aac0358a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/fe1063bbb5959097754b1803f420df04692d2030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657169.98/warc/ff76f12b50d65a44e3310bf01b14ce70b33c68ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/03c82b05e0d62b8b8a8b59ef56e38f48e117a511.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/09968dbc486713436a69af7faf29e57d7d4e7063.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/6e561b9b45781e00c2aad4c885963e1af8bc81ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/a55b510fdb1da863b38bf0a2c63d6c81a8760a12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/b99c808b094b09e17b306643bf08ac2899888cf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/cb9311c31c0898bdfedd9b5c101610595c0982fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/f012f6773dc0166cb9b642af9594fee4b81b491f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657720.82/warc/f41cbc695b5f1ab2de4696b0d2c94a55dd0c313b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/07bd4d7f3d9f244c82dda62ea501e7b0dd0e12b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/1ae15ddb26ba2f982a968a915e3a42370581f276.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/1c89b5aaac0951acab10a9a7eaa5d35dbe3710f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/b304322fee93dfca9bdb83521500c728c2ae63e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/bed771bbe56d160682991185aa8ece24c3d47561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/d5e18438f1642f7483972e8b9ff0b26f2ca62106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/ddbd4aa8c52efcc03f2220e51c475a617ec18f63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/1685224657735.85/warc/f702162d6db2249df46e43f7dba1c07110e9f92b.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-23 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2023-23/mix/input" + output: "/tmp/cccc-CC-MAIN-2023-23/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2023-50.yaml b/configs/cccc/mixer/CC-MAIN-2023-50.yaml new file mode 100644 index 00000000..b96d12d6 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2023-50.yaml @@ -0,0 +1,1014 @@ + +streams: + - name: cccc-CC-MAIN-2023-50 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/0cecbe8f92b0b7a2f66894ab120a0a618ea81dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/1e7b6c1112407bcd893633999a216743975539e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/3f45f62c154b843e6cd0d2f5327a6623d2859925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/57b3031b1f4608c4ba6236ff0aa7f3124c186f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/7b3430f1e45a2c7ed0787d6644461193f9b9814d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/b97b6860f32424db8088805f8c54c3529e847138.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/e9fc88bdc1e90010c6f20b2927f3a3a9f0123e93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/eab8868532937b76b50decaece3c9b72ea6c6075.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099281.67/warc/f2213d2b8ffaac9e85aca42803fe606417ab950e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/22d1472f4d6e5885df94fc6d2d135898badfc8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/39047a64e5ebe8c8946f96cb3eb32646c0d20e03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/4cee62c76b14dcad929b3b06efa5ff736e170d72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/500cb8ec51aed4a55c11e02b4a9cc45f9d173648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/5340f10a81aebfef231c9373e9131aee0f57f028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/5eb6790a991c2834acc0667a9912ae7dbcc27417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/a50280f04ffa79ea003e581299f9edd6f4a07ced.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/ee10765556b3f6d446f26b1ff54493d9bfbd53bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099514.72/warc/f206af06227e0fdd76b636e4216594e49d178e7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/28ac08a9878370ca64a65cc3543a7554037123e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/2d01d7997ca851bcaf8977325775900ce5ad8853.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/3085b1bdf6652766e9f05fb4565d9d6a2ff7ccde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/335816ee0e9240ba8f318c6775151ce2f7161635.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/5782621dabcd67795f19791a747f4850d330390f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/5d630c5158e86c66dc356103be407ade3a13737a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/cfbb14cc5803a52db4b12bb5e08ab86a815e35dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/e2fe761f30b7d12e2b546b87f314709aed534561.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099892.46/warc/f722e65a8c0a26bf7809edef55f01ad724a11ee8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/4689a64f6abd910bfbe32a239ef72d75e699a508.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/5b0b17ffd489fece2d5e0113e199bbb964aefb59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/6db6c3491f64fdcf80b14ae7d714a360f4180afc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/76cd1a431cf60207cde7f1c4070a6fcb7ed0d1d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/ab8c8109be781fc2ed8f18ad32559f58f08c740c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/bd8767860caece5686c792a2a09d9cc11e75610a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/cc1af3ba2b031af1b16e40596c61c5b397d630ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/e0d7703220c909f9a084991f6317ab70a7de7eba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679099942.90/warc/fdbc0701a1ef72f80bae8729107f062db28bd15a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/2bd05b91993150e24fea8e7e2add3cee9bdc2d05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/3ef246eb2492868516644fff9aa354aa199b7720.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/85e3b681d88c57dd9bf61b373d6d0637f681d5cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/9f10df47f38448777f6f563cab6947853e813225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/a2ac36cafeec9d8c51039c97ace2d116dfab5393.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/bf671117d09b9b5fc79c5267951ffb641abed024.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/dc86e476b3d3735734f94124b42ea342817524ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/de6bdf2eb5ddc1898b9d9d42cedb6bfa126cdbe2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100016.39/warc/fccffb44a9d9fd1292fa834795bd50924759fc3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/00afb54860eef830e59fc5eac8cbc6ee813d4905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/0e59655468e7c62ce2ec563e02b76f70ff57a1c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/256439042088dd4d43334b7cea19fda47820c870.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/48afc92dfdb4fb62eb7a7f16e19d2b45f859c8f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/8aed6d07adfe0bdedfbb5e045768b0f2580a33a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/9e183b4c5b27cd84e9b000830cd3df7b6a51a9bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/a883122994131ab0f127a4466b9cd234c204c701.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/bc1cc4c23fd3b40d79053fa4a920bda5f69cce44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100047.66/warc/eeed21e3490d1b2e65d3258421f8137c694f21be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/0ce6fa1211f623eddfd463e4c004099df8bca156.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/17f00e34b7abf3a52f8901ef73c0223e71835368.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/29d1367e92e8e03aa3cb20d93cee205ce81a1a40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/65ac74093340c276ca3ddc5b2debc00521547084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/76e486d925d65b4722600a8dae9135b8d27a5a71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/8432f9e31723eed2cf32de14f130101be851fd58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/8f9d31572d113ea161354628b89e3813de74ea42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/a2e86e1bf8aa6871067fa3e63eb0bec9d374dff8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100056.38/warc/e4b60fae81ff081c6bc80447aa7562ccc33b91fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/2c110b60ef4f480437f1d626d4d0c7cffc274696.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/3e06db0bf8e32bf7d3ab17cefae7fb917f2bea1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/57ba017737368507fe89c9999164abf46a1690e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/5895788e4090eaf9f7e9046de2c6a4a1c98cca01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/6d8b9380c8dfcc49317b90b7118fef9200a60dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/b3b8c5a978ca558a0d9173601e07b1dedb3b60fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/dadf047606702586c58004d455b77664a15405c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/dcfe5afca916b5108f7b0bd1c05dc592b13b5bde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100057.69/warc/fc56ddfc8ba4f6b637de5e3a9787306c65ac0f5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/220de2e0953d1ff6b029e9087040d0fcbc44fb8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/36cd71af91c68b04af54fc65634259823a05497f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/393d10ea3779602f66875c2425139cabff500388.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/583a2ba70a56d7069a2b5203cf8fb6be0492d68e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/7f3995ea410f7ccd48a7dff92cf5dbb6a3c528fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/b7a4b27da991091e2839919bdccc89f793938dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/d39a424126b64689e0727ebf3cf15658145b07a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/d7f2bad6628b920d8d0b03775484916b0257e048.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100081.47/warc/dae6f36306ef2e8b555cd185e80dfc7dd537c0ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/017419c7f2d0fb85753de1417d6752a6f74bbc0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/0da33b0d75511876a0ff075305797bab7627c3d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/2c0179c59b16a7defbc86e71f05d702777daafbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/33ef30451617c5e32e0d443e3479a5e75fc91c60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/8675cef79c549af2262232258548a5215537d31e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/c924ddc1be0636f2533ee261db9b215866665854.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/cde91789efa6eae07497bdf2685f162274dc9865.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/eeb19054a4cc02066b35170bbf5082f61d7f66bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100112.41/warc/f94bdc53f880fdfe50369bb515bedfe125e8679e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/037e6334a718935e9fbe8d87041b1101009b4681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/0f38915ab57e913aa024305512b93eb6035c1725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/12c5a444ab88506b0d1e5dffcbf88601ed93fa3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/3f7b402dc89fdc59faeaf218ebe1603b9460d301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/62b584a785ad362c29e83c8447ec1bb43866d176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/69e61e04920dded984e6469d0959003f85ff9a72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/8a165327b7918cb2c45e0327e52e8ef58d0314d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/a6a245986491af729d0d400640e2634b59e95357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100135.11/warc/cb1d560484ab4b232f9a86d401ed4c136bf26d4c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/1497f5eca20f87873898c16f1c1f3327afba8558.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/336b29528e7e728845ca973d5c2c8645825e8816.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/370cc69879e1c449a898faa58ffea3861fb3fd8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/3a9c34197f381630a8dfcbcfc968e1659b3105a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/83d9722060e1898bf488300075b3b66162603757.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/916d93fe4fe54b8f28f5e0d256e205b78e91b771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/b78d570225fb2877cfc415d8d6fbecee53949d14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/c577b6c83a5c3d5b35210b5c29f6ba964cd6d0a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100146.5/warc/e6a8e56e0f662bab963e9332a834fb6150806d6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/0ccf61350dec980c9b36d2b464e5b2ad04739735.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/4e6c59ad88fda65b7aad1e33d05b1e08b4648a75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/741eac6b682f774c53fddd3c3c2d72d095ae5805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/8f0bd8b076400d0ccd7d0b07ae7fc10d961dc367.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/afa00716f1d3ee99d63e678a4b8109822f13e034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/c0e0eb28974a4157a4c5f7a68fcd1476159d5909.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/e426e333bcaa8bcf781d8123380feea0444d39c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/ea2bfda629313964dd73d83c8dcc736429157dea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.15/warc/f4bacc5be7d31d0d7b614fa502abb988678ee142.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/870b20d663af912e5ba45c9fddd30167d5496bee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/a249cc1712f200d76a4e37fc925fb6fcd50f53c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/a336b0aab8c05c0f2c839c2d1158256cae06e69b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/a6ee465df52e96ad9285f84f2a0c9130bd3e9b2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/b7faab8a6e5fd84da959d54af109b5563db15807.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/d32b4921f0e3de33a9fe7847af1884e19434729e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/e5824b4a5942be6c6489395526ad578cde2bde2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/efffe615107ee480846fb6e2c19da5d0757b1c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100164.87/warc/f02211da6d0809ed15405f4ab74b6c7531896252.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/2b850331d37d23f5a0d7b2d2e810cf76520d0b93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/3da7a3f77751ec1fefaead010b72dbea0243091e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/40f56c8de0ddf12286d43f115cb2d564d83c2a78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/64638afa3a7abcbc2270dea339ba4ea28b64b5be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/77e2b8ea16b80f5a5bcb97a978cbd91c7ab47fc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/adf07965b8df6d6bf3a4c4788654ac5d7cea4e6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/bd96b239a858997f8bfa76e374ce57c5de584845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/e50724c43537078e15e3360f103e21920d645505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100172.28/warc/f3ddafd1719f0f8e7311cc59b52b82fa5baa489f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/0399862cc6ba61669e952277ae16d2cd1bb83ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/1bba828c8e11ee2cddf9d65f1976444b808d456b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/4761e06cdc955afbf6d269cfa8ed0ce923bf0ff1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/62971bc68b7a3d4874ce0e99ec59ef00d924c044.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/8a88ed754ed7f3e44084be21eee4fbf7351f25ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/91f1c94ca5ceb86f8bd5bdbae764ac3325de2e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/ac4b982ddbdc4e07d25c0b3372163352e430dccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/c5efbd691dea8f477a5df8ab572ecd0e886c2470.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100184.3/warc/e0f573220dfd8ad8fec44796e8d828767069dd7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/1c435912da1bb01a1760a4494d0b1631562064f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/254891d6244e5633c59ada64a7dbb524b35eedbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/3af46f135383faa1a641d1f13a210e4d2eca9724.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/4f89e56b8d14cc660ea71b58f6bf118532bb8bb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/663e8ee7d0099dad2483cfddaca1f97b95a695ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/764caf1950ee39c57ac4f2c843a78668c1bd6901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/8b916baaf8f6fa6485ff1e84c3111ff52ca95698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/b05da1304e227703fa1c73def7f5d7419f668ecc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100227.61/warc/f2fd89cac4f718c6d9e8e28a7c8a4d8503115f6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/54835d45c5b3c232e360baa9ae4c542a8822a01f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/8f8f2340dba509870538f66f1db379f6683f7c23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/a573cfc87526414f745aff7aca09a682e6fb442c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/b0009ceb86ad00179e2c54c78b68cd1162abfbe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/c55ec24840b8ec69b08618406537c03d46c157f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/c5e51eabbb62dab423549b1b5d125e8f00bfcba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/e05c9260c2572fb69d4e15512c0cb28c1bed494e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/f0fa9d20cb8754efb4d988b73d3ea87c23c62597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100229.44/warc/f81ddf09490fc958b99d76bd6f11d99f51f87670.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/0bed91879863eee5648f7365b646ef32ba9705fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/6309cafecad622a16663f9426952fc20b04e81c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/99ba782a0727a450fa237792b6c476c1a467079d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/a825eed211b4ab74c3ee311803903b450e243f5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/c0f6d2170ae8b34f951243f98ec7c49be43b1d2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/c30b4cf7847bac5d774b4d9d10bbc9e3780ba395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/de8c65bd73605eb9236d88fc479e5d5d63cacdc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/eccea2d51f77dada79c40dd2e120a98abb77a877.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100232.63/warc/fd9a2d03f00b624f9c8cf59e4d3af6eceec0d7a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/3901f91488d6f96d9c21da10c8de315e2a0e09d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/5582d0178666b811d709403ca412999f665fb079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/7714db740b95788299fbbcf60aa4a10e90c86734.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/888dec5cacfb7c6de887959a30360346c171b507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/8a186c01fb0771c548349f83e95bb4c8cb25efb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/8a35badd18a21b586290f47d0571fe601ee11b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/aec6df89ac2831142692308bd0e9776417b1131b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/c03efd838fc72396d08ab97c862b8e5167dbbbf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100258.29/warc/d14de1b7dc6c5b0d25ec923ce6d20e4ac28cc6d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/1e01de38bfeab825658fa5004f4ec8fdf0dc93a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/5664d795aae67e337d133ef1b20ac787ccd6fb69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/7f32909ef5895cdd816002ac69712c6d2fb8c112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/85de47b0b75b4f3e58b8a96f9dc645375aa5c691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/ba854a19c1c8465795745428a4bbcc8fd7ca1dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/c229f9cc72dc33b57c1516a98ab7c5f31c6fd2e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/da1546033cfc94ace20eb9958776ab1c5a9347af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/fa051eb6c9d1edb1d5efdd275f4a748f7e78b9f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100264.9/warc/fdc4d786347376f4eb1dc49332c2b7ab625bb92e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/06e8a73a2a991d710305a11324cb5165bc8b727d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/1bd26a69cf89da5ac11ae02e152ded5e0e11d63f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/360d2f4c2f54cea2f10f7f5b6e24db7ca81a67f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/466b5fa05766374740019f07d7d1cb1f496eb618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/59555cf53a4a365a259bfcbcc2640dce9095eaed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/8542fa2943d9a2574ba3b8bb97089ea89aee1aae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/8f4c14a53a50448c5438b6e2f53d2460a6f59253.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/aa1a9c92e2d70be2af5221db764306f467e27da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100276.12/warc/e6de4a75a2ec5bffba8c7256c4fbc1a9db56ff72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/22a908b4f265d69140a9e19da39b79f23046602a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/3548f6b23abf34cc663ed85c3a60b04b9fc44531.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/49bb98eb661cad7b236d2023110ae94cca677467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/57dae09e4e5ce40d689b90d6530624be2c2c4a92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/7585599f31f5440a2a8bb07af79952af25cf80a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/8403ddc21ff77528eff4464b340dfbffee38e9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/aa4495e4c6b46e6e20d7efc42fa5f12bb0b9e7ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/bb221e716b768db49d4c8fc41a8ae9c4c3914e90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100286.10/warc/cc51999f2433280f3af3de08406b13289f8c06f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/2aa69a8307b4cdf8559190dde4986a2bb3a1c8a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/34a042bac84f6abb4be3e055c0811f93dbe0f66b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/55083742638a33abf99531404452a429ece45924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/655262ecb4f2fc441ce98e072557bb308e9d0eb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/7ac8753b65a5803e357d817b06181587617942d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/b0ee0cc77064807392a46966d1da4ee7dce72518.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/c9e6809e648f2b8e6326d579ff2080a9a500487c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/ccb7fe56377afc9398229096927438551e477704.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100287.49/warc/e3ece367e81f39d19badee4a2a7e04c86164bd5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/1d091450b6aeb81c27dec647d9ff619de5573030.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/3a0a53525dde01804cfa71b8cef78dd45da59303.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/512692ee7fa01f0603a65c78214e85247ab5b8b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/6989abd47ccb708260d067494038b759143c9401.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/6a8b898f08a3903ed166abfd32ac60add6a06fa9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/89965854e38449644ec0fe2e1cd1bfdeb413cbb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/bcd6d1c2c4c66c2f0f3c9ea3d77d58429b993b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/c7e824f672322933d8045934c4a738bcf6e2daff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100290.24/warc/fdd15f4bccaf758ada8238f3dbc0c20a7dd78124.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/14d4b824c3466d0290124d52d72398014417cf9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/25eb8178d2fad6a45ce0f16de4f6273f0856dc2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/4998a1ef79904643339d6b0bfbdd79b3336bf6fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/71913081457f541721ded08674774f77566b4449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/96cd67ef6762e8c67e43f2dcba142e72e5116476.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/a0a894d0d33df0ecb5a34a925de897e4b42d265a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/ae51bc716beb9b54cffb988e641732e5271de028.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/c81a2f69f2040076246b4ee4e4c6e94d8c3203e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100304.52/warc/d29ff63226a78d86d3f627d50f4d769b20006f26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/141b7fb010101d1c635626b2586b158b9bafe410.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/285e405af4597cc289cebb0b9d66a3877ddc58da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/4dfc63b2950e29b9792a6079c0662e8a46e05511.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/7959a2f8a4f8a0ebb5ba3b9d36f875ab351c3673.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/79f5f06c3bfc6aedb20ae99ce4b433cf4858f012.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/8f67a9a17e9a5e56b7dee798b6ee481a30615005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/e1428b2c4a7d725caa4d1bfe2e962d1ccb2f5671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/e5d3e78f71a125e05023da0d9432aa3eb52cdd8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100308.37/warc/f4d75cdb964ed7ab40b3ee760d737cde61c68013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/1112895194c1295c7a571f7ddab724b26727500c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/1764b50e7c29aae4cfe0de9d9c5c4d770c1f7434.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/1c3bdcc0e09e52cd9a68d8d531447bd7a98969a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/474d03bc2ab2a1cee9d161a112e8a9a4009d4c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/5cf5a1408464789be02b63f80d8df2e347cc254b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/7e08f7513023cf61b1dc6925cfc4199c3b8bf478.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/a737caf69185957fdfc24482469bfea233e76c70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/a7a73b4f3e981ad179e64d178649f3bae11122b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100309.57/warc/e501532d81109a7008bcffd096288c42d0f39260.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/1f63f57a1fa84c8e84c4f9f4e8916e6e1e69cd54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/41df3597da3fb0391f17cdbaa8d4c59076b4256a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/5ba68fdd5e4ddb957a323995392b5c6e26fd8b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/5d177aa630afbb271cfdb8cd91e6782bf0ef2c5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/666d88ef50ef0d225eb23c6d1f827ae2840cfc63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/cb881e04a36f33e3e01518b7c2cd5b5ef35a304a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/d0d4304f588db932db59031a7588632ffb6d66f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/dc898b8b6cc5b0b74a6a53d6f9feeba101055e26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100327.70/warc/f18f520a0566936ab9e6150a91d48c817fe3324b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/039d65515e1b2d16eac15054cb0f72f9e5b19c2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/24899618f3e50c058ab42754a1c1e1b30a2aa8c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/657f2e20cbd87d4530aadd9726ce3540c3a6b70e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/6c81e039edd5971dd5c7ca173fbc0b723e7ec73a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/75b6b0f41c3c7329c87c05821a4b0e2010eec3cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/a8def138b4fde4b9b01296b67ac2074b0c7fd2dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/b92bf46326d4d12db93a3bc834831b59f7a2a53c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/c42028824ca149ada1ab5234925e9614b5d92aa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100381.14/warc/ce5dead935b4d966f7e81392a09ca835df25e763.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/02b4278a38d0a197484e2f42bb858d329dc9f70f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/03d7a73af1ebab649c5e8477ce5342d8e0a459ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/0abb5ce7343a8b5b0b350787aa041c5f6d087a47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/2a2543a2fa1dd3c852f01b0540d85eca7baf526f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/5fd8c5b7de627a802f2d6464218e5d0dacb1214a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/69e269f38bfffc484a38733ca00183fa7cf57060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/6d73c9bb60e9b3600e2b69e7ee31de2084fa5954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/abdb086f57319b96c5c52f96bafe49125d527520.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100399.81/warc/c1f73ea2517db91eea0f39186e80f10f5a933f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/036b3d00055542c4377678c060730506111c00db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/0418e80663d38660cd34fc543fe48d1ee1d2466f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/05ba431b00a14944dafbafe5629857b2ad222077.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/077382d87cac1faffd2da8cd3dc65c7718d464c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/0b84cf1f26d9115c67f9d31925399d3c24b595df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/3b12b48dedbbcecf45de3eeef04bc1f2e4e5bae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/54c875606ab15a96ca3289014e6693dac9591f28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/73adec8f8cfb90bf40a7775c443dd11ae11098cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100427.59/warc/ecfe912d7cefa8454695088b92017d0fb5131f0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/05c81c7eaed250e255d81c5b12bc15f5c1519160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/0b401f8d19d49b52e3fa8a2846165a96c42c83c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/35ffceb70c8c0e57ffbb825c8f03c631a7ae4bcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/38df1b56ffa12c71b45ae805f22461702448c360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/3eac7b4aef086c7a7fa8932d5acdea934e6925c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/69e8657462930d7d932a4450d5fc12f9474f9a94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/9fde4897af6968a354dab6a132a44f9289e77c09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/d003660033853a5a1266cf081fb3e83b7c2737f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100448.65/warc/e25415ca637ddb1c34c6a172d4c00779f211367b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/9ef9394f85ace694f0726cfd768db4590edcd1c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/a05e49964558d50c9c188dee721283a208a306ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/a3dba0f68f6af45575608868d8e855279aa3b684.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/b5f244909b998912c7dbb7a4247603226fe2251d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/c9a6eea65a9345a4a2ac087f7b5e99fcf7d958e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/ce62fd387661be9ee1d394167ae2290fb30bde6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/db5847ef5760d46575081f64d17e098f32c83000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/fdca3516fd95a5a12aa0f83a52604d665f2a2967.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100452.79/warc/fffdb8c285f8f3b613f8faf67f211b66de11b37b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/037416d99c99f7516b56d232c13afffe0b0a7125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/11a9d73d293eeb2bf80f6481c5bc1f34c1c0cb11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/535a11f308cc372d2d28cf80e9c504f5ec8b4c9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/58fc809a0b6b7dc2fa67fcb58e9a89d79c11909f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/6cc7696ff5448ca0398269089feccfcae3ad3302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/7c5f44c8509bb4891ec12642df2cbdc97f61af1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/a2639b5adba1c44feda3cd09a33f5ec00eb6ba0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/ac418e62cd0a7369cc97d1b12bbed18f2e92bbad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100476.94/warc/f8e861addbc38aebf38abf2a1d2a5b9f1d441426.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/08083357db7fe634c9d7fe0b1ae6ed27b32ce464.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/086ac0ddf7858a62845b2586c0c3adfd2145b403.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/69a83eeb883a7d516dc5d0359e675f3e5dedc8cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/8d1bc28197ec5a11b6677ad649f12223898b7d95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/91dbcd4a18c82b6d1a65c1721a30deccf9e37a26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/99ce11879854d016b10dae2272d599a055aac4f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/d360f375f8c683ea48e404f895b20ba65d96b397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/d7bca81f9ad2e635c9d2d52f5aa314fe95981abd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100484.76/warc/f965b206e645ef88ef562e7f8591a3669a510524.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/049961b3429cfb34ff2b12657da2cac1a32db652.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/5f1fbd2a21011fc114d882c5672b1261662ddaa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/63b13e41869501a51b3aa005d308151599ba16fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/7b90893d2c0869d75cef220b3d657edd6fc1561d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/87701adf3d889275a3f5c0e323ba74cb1ef5d5eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/9620d7bd3b2b91d4a1787b3561fcb22fc32b9087.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/b2716c98d8ea767460d8bb24ba069994a1552885.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/cfa7b81b5bb5859879160f7a9939ab54a35e9628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100489.16/warc/e5126e746575b2cc71e169e5c8549dfd144dfea6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/18db502960cc4e3dc0c1160806853898b68005af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/26311e18d27c0b43dbebf7ca4a4764604d47a7b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/af16d56ccd169ebebf95644447f1426ef0f14060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/b163aad5301d9e3538459b45c3f5c3bd31325bd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/bcebabc6e7f68c7edd04398f2f7d72d4de0a66c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/bde21d8c6e10d7ad1f32ab1ec3148e5d5b5387cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/e0c8473585f9210ab0feb15d791303a5a4fe0977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/ebed4e1446a6323906961333aebfeb7f282616e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100499.43/warc/fbc6ad166477aee1c5dfdcfd5b62d95f03ce0575.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/2add01d7798379864f161a0bb1b5f305f59a0ccb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/448bac4fc8d024519938347c9824a2c57fa3aa58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/742067d17f5127f91cc484ef8ac56aebd773a62e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/a16eae2640744a09235a717dfa8bedc3ecd62af9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/a595400a484ccb3c60ae7c1f9019203117b56676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/d61e431e49593bb09b30f3d3c2c57fe5d383e52c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/e2693a15c3c42b396bd4bd4eaccf9d3a0e0650c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/ea1d8c2026262cf1dfa56b6d9f0fc3b5c4175fa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.23/warc/f9590f35946e40a80656eef18bff36cd512de712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/1e5d9a37741cac313a669eb331e857354b1fec42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/651a196ab3e8d424d961e4609e28fe511791f897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/6e10d9f87f96e0dab2dbe64dd14028f95fc14eed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/8addc113013a88ceb9dedd7422d8905ed2029f30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/8cf735d3b3aeeebc52100e6db3d9e0e28ab571cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/9a4d1929c90c34007dd0e7d9149fc3c135752b77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/a095d97831bc2c984e8b713ef1655b18636ac2b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/dbf03fc05d6f00336be5a315adee109b2d362a17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.42/warc/fa284cb178d3e38e924c22a056d9e43168f85d94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/19982ee59cb117bb426a824377bb3c6b262a66df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/94757855a4995e401ebe07dc8822dad8aef98be6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/a2919f3a0da5836adb47404da0c9c7ce41493bac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/ad6282d02b126059c3cce36f4cb1159a1a1d9f29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/b5460816ad2f67b48c0b6121246328d9407cdcfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/cd2380d20ec210fa718808ba5b3247df2c39a771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/cd9724fc4f179ad01e1d77a3ac0c311ae15b6613.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/ee68414ba99ef1a5985d657146ef48c2ad866782.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100508.53/warc/f725736403a7cf6a2690f45fe06b60092680d194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/307499a47fea51020e742fc5d6c485f6554fd669.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/620baf7278b0dc0e094310a3200418407966710a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/814be7e1cac62caeddf0f251206a66e2022adbed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/880d0bb03a318ff05347d3570191df3a33f3d46d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/93a0eb7cc337e4e37d3ef33cc842dadb3398d75c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/995576999077e3be32c78ee0da6f765931d314d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/9edd7a16bf0579316747fece4d5c33f350d9dfd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/ac4b070b73a1e9be42f83876e41dc333073606fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100518.73/warc/bd8d23a81a782cbe4698e095a0f7c2323801a639.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/28685604e467e76f2f2c6d57653397f66b06844c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/47284d8c98c93455226f18793ab393e0af8556d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/5c7b3d5189b84bdc8d470540a2dd5d79a221c274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/86dbf60192723603bfaf9b7949ef88deedf36aaf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/8d7a7ec87b3a3c7126310788c7872d872c9be25c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/b0a3c63c041c511f10c725ee541289974ebcf8b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/d65068fc327bd48ef85605f973b177051e3823b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/e424c3a610b8ea4647460c6b677b1726cc11d518.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100523.4/warc/f594bd41585416703583269c0695f73fbebdb876.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/108938a265b54a4746d38d0b768d4f3ebb04a8ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/279becc5aafe62b441bc9d46e30c047eff89e229.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/307de43f9087b8fdc8afe17da13306c270aa5644.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/56f42ae4cc38e6edbf1080336d702eb438294c36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/8b50c54a7224cbcfd80ce2b771fcd9723d076e16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/9203d8600506f28f35712f19675f6908b8f8599a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/c36ac8a4d492ca3f47e3981321a4db2908305d69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/d0d972774aa33ae1685744359074b4b6f8d786f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100525.55/warc/fd40908672de67ead916fe61bed723ede87690f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/594107a99c3f2d07bcb3bdd0d0de984e297654ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/59fde9bcedfefa5927ae1ab7f92da5e8abe1b51d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/6c57bb5191ffbc07d9c7f5acec6e01d1c46a03ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/c05fa01c6c536c907148200b7f846fe66bbb9339.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/dce102e36363a6d5affcc72926afbf9d26deb35f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/de412d8931b67d6628d24e770d6ed0bdb843f3cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/e4baff61a68aa6af5652a74288b3504231783afa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/e599c51bbed9f90570af1ea2f30c21cc80b6babd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100527.35/warc/fe38b864d96439b29e1fc1b79c669dd9ba224b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/1cf6b9c60abbe6dc6518e2e7db7588e1856e1860.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/1e9c0362e411f79548d7e86ae90e72c33e16a532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/2559053a2c211f575fe0305204de7f254f007f59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/8c1695321aa841d775967d9e7caf9c358b47adad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/9898ca46566085b422d774a6bc74fe049e8fa05f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/ba7300237158f629140d66852d5cb9c2d3d300a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/c6faf47281adc7a503fafbccef84d7d8f892a397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/c77904d689b32cf0bee8fb36207c7e65e55d9f84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100529.8/warc/e50fe8628a7e435e73e3112e123a32540569a390.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/0254331c71a4691fede3e858b8101c907dfcb5c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/56bc1828cd975ec9eea4e85a0a7b9546a617f29a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/5b994677c5231c6cbe7ab36aab6587d2d71a62c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/898ea4a7349c6929cbf3e6b0393b0596e8aa75b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/b98904613871051d7308c3a124c10c737c7f8db8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/bf1acfde3fdc2bfb8cb765344aa0bdf53e7d1bc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/d6ec78e1075ec5fc5269ca9e27a044498992e2b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/e7fcfe6e669c53a4e834c50bb0bf70b8ca7e672b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100531.77/warc/f569cbf313d3fa00123a4d63bdb9a7b10681062d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/4dbdb689c1728fbc82f226c48dbe2695f3f2d8ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/55dc5f5ce89e2b7c8e8539623ea79735a77e5787.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/62a14331e55ba68fa944b9e5fd8713aec33e36dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/6e2e9ea1192c9d2b30b2b7cc49ef04f5d20aa7fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/84f542a6fcfc4d7b5cfe779ab74e07399c384124.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/95cf7228aa823a23103b0d29364a62dc775400f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/96d2e089b0f0b2c279d3713eda4e003e5b0481bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/dd251a59c84acd849268b2254e5368546022990e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100534.18/warc/faef3de3ec46415268cdbffffbbdfaa86a96e492.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/2e3dc313edd134d0356703fe70233e6023e05fd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/351d312c8d24f9db98e5da6c40f14c880d609de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/4689f80debf91e620cb07630356e6dbbdde12df4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/58af8845dfe173f117669525444302f105df09ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/70f48b70e539f5627d5f949ebca48a24930c7d5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/9ab445a30e509b879cb1467ff17aeb856cac95af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/bf632f4155d619dae954861f602679a7604980ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/d0bd02e36cdaf08d39a9a134acd2f634dc216525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100535.26/warc/fce680141d1b91f2de3f1b4ae65b3abcc04a4d43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/38f4e26c46a0b105195c974ad1453749f707f834.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/3b48558cc6d1be3fda15e9efbc9f39c065b0f8fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/3c6bc7cd5ccd695d2874ed38c78f4da98c117332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/6c595e0398cec7f41754ef6dc596c6c31eaf62b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/754d46ed0391f107c0ccd55488b01e166ede2027.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/9dc4dbed3da7217705c0b8102ca18127d5f2cdd1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/cf59b11d883570ff09e38ada55c94efcf82c407d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/eb36111ba059ad7bc44b195579656ea7fdaa9676.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100540.62/warc/f4ce4cb219962c3f0185e4e99aab92b2d930e623.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/060b046732c29cbbd71aad1366cf573000bc84bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/17bd1b91803b2eced2bb06a84aeeac38422ab000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/1952e34613df9f7f46c7135867982afc737b51ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/25cd11f2be8628990fbaff929e25002969d29998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/3d27ff8a0312dae3078dcbc3edc62e3175bf187a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/42883e64276fb7af9a04eac3644aacf31099d564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/6eb440c5a69d821aa91ec0c5aa170bc1730098d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/b31a14731e2344703f428872cb87ddaaf5fe7318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100545.7/warc/e84746e737592a9b43bae39209e0d7c7f8dc16b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/198392ade2cb5522144f2f79df93ce33d9d88742.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/1d0614d123f54f1e6564d7326de6378cee409be8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/1f6bddc3022e19ad95fefd65ef9c6ba44bf35ee5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/5580a89140257f0746149cc08d44798628f6253c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/6b52099d3f2cbf60f84c8ea1574ccd41db56a0df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/90a7affe1867d44ebf784529476efb1032c70d9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/b3b49640e4ba00f8af41da8d90d6c284d16fea76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/f4a582b0837c5bb9e26a8f0ba0fe1a2abc99f764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100550.40/warc/f926c02552a81f0e56a9017e4d766296adaccd5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/0c734451cb26e5478fab2739fa27083040b94674.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/24e6add8255c7665e9d8b36a744fef1a64509022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/2c8cd8a3eec240945c786f9e205cc042bf58c83f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/446aab4aeebb5a802169a6bc76d7509064a3a9c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/4ac23ee4d14487c9c31e9c04628aee2aeb46efd9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/825e77859a3abeed808fdd010003cd6b7cd5f020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/d7e7e366699bd7a87ec137df3d09c72ffb1a6329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/e330c9b2af3297be5955cb530919cdf02cfef662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.17/warc/fafe81ea1d417c40dab7598d43b23d53e0744cc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/19b55b305e97abdca5eb3341e9dfc0d2a1c58ab0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/1fa7b1f3f826d02917c178461ef8b4046b35be13.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/2bba6388c606511356fdb4bd25ec74a5a71076a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/41efba287b9556bfc6e7c49ac38319b555c470ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/660a53920db12915d8eec90c3399b3e1230948e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/8b7071281d1caf82cd004ea7599da24cbe946455.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/a3869cdf8f1daa46c852f996e440a2d4dc169c5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/acee031df070a17b84b9d776c15650c82ded436c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100551.2/warc/beebb6f976fd21b5fe216fb2264fc0e1341e445a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/546b4052c33c9d8e2f3f6f53849d4df27922de1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/560c9b6884fe3ab548e161cda858afccb8426f4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/7cd4fcc12ab1f8a0f77a58964af63ca2e335dac8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/a670afcdcccd1468eadf32ec1d640c922cebb39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/b6c76c8281af1eb94985243a13a60288ba1c2fe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/c2fe4d2754cec5b38c060f03a2cdcdd9f9dd91e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/dad5ef4ebcb048aa64df199986461c42855af927.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/e46ba7538e0ce0bbb9eb3787969dc84cf958a0b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100555.27/warc/e8e6cfb9e96a571823b3ff15150aea85111405f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/0118d5658322cd809b8caf09128c484b19ee37aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/01a87f0516e68793d93e54706edeba380a6af6a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/5201a0c0cb83ff0fdc0ecc9eded7e631c238ef37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/6640666def1af7f5011dd04dee90b5f9c6032aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/9b4ace88cacca091d7aa57594ea21f60b12fcbef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/a9a63fe80dca36e394c2674f33555f05b0221767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/b4e0cde2d7010cf35e55bc4b2a1fd7e4476856e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/c7a008517dc7d55064f79d1aef143eae41d23981.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100568.68/warc/ec631ff5f7e3bd759e5f616f54789d3cd0a6290c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/163c83e4b011eec3d86f9e71169acbc570fcdb56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/240525a60c33994f6f1fb9bee9b53ca0206921ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/2c1bd8dac59df0e1fa78999e0eaaa1d20d03e0b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/6b755413da01ffe7dcb521349e27c2e9534160b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/8f24cde80031e70615335f83c1f32c5d8d6033d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/b21f61a5acc51307e179f6db3a558109f3f709d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/b48a80c3b36821eb2b7e27608620ad32041d2421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/c595b0289546c049e27c24f7d3cafe2e7edf2577.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100575.30/warc/f995ed8dfac3682a934c60d1754be507f85c046a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/0120908f445e46657d700e30388d9b1a582d3850.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/1c64bcb5387b104fcaa0573e0cb667fbaf87cf85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/48b655583011eb9d8aae9cc1c7ac1f55a2ac50fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/776d1b082c43aef1fbc7cff1908dda6f3a693a27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/776e918e3b6d9d760fd9a98439328a9fc3c372a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/8bcd4429cd34080138365263d1148cd58d8ed234.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/c04f1acf9369f520fc7fd0ce9dc4ee62573ce6e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/d45b1eda24b3620209ed7ef750763b4b32203a9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.13/warc/e9a502cf7a199440d639bca77e2b18f85dc66505.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/1751454e60437c2ae8ed1876c3b6dc2057f3e33f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/1cb85305893fae1ce4dae3eac862c8b6a759c9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/29772e2588afbb92a62b41d29761dc3ee3076f90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/326150b08099ae716772b8b156c1ce82122f2963.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/3881e1f2afe1a212be89a0350de99ae687176c20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/7919b4baaaa30611b81dba30f4e7d31294979b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/a91a562c5ff172dc9aaa92752e7e9b6509f5c748.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/b095a02424ab935d36706db9dc52c952c76402d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100583.31/warc/ccb4840ef76f1b18021a4fecafdd57dea26b31bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/1926f214b770035000c2f4b048c18154a1fc6f51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/1ed7618e57c1a117c107dc801f76cf6be731bf03.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/2a4de166e7c54768c39fc357ace56aad39d2aece.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/4adc19b003bcba5052095d47e73f1c528734d290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/680daef1aebfac9915ec51dd37f58975b0dff458.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/7d1d0a3ea570c1acb81630b098ba8e1b2d846f41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/d4d8b9b68b1ff825cd0430325200faa063373c62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/e7f667a1528932555d384760496dc3a9e2470564.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100593.71/warc/f4307b69f7c86861423fe1c5038b33239cb9d4c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/1c1dba8ea443df4a981a95880d3925988bacc51c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/28411118639a79ad641ab4e6e0381495bebb9256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/2cb0a91321eb139159cb67e04fef6845bea1e397.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/41da3c0b3ef2d905f9db4e66dc115645ea5ddeb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/538cae2548beed09751e33c505f1c2e915bd3fad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/7c20dff5f9862f7eb49e320677af317d3cccb7ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/8ba778293542ae6831ba292b5406c46549ab8f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/9d95ca8030e4b48bdf353c5b862d2e2694551642.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100599.20/warc/eeca111cdc092367764b5233e4e127ccf85e6060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/13f4e6aa10526ee01d5ad056a960aed1fab042fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/1924f5e1d13f9d7806414b050d080f26515570a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/646c4e447a23450aaf8a9e9523140fbf85d15c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/7c4e9302e40fb87f6a81d2ca153c9e67c95793b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/97e4f5f9d81d03d844e60d8f19433b93dc66d02b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/9fe9a4d847616200858ced5ec7963199f678d7eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/b30bcf6a509636c21e1c35e3233318fe7ebf86c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/da55f1247b927b42669b8ef1df7a2f15e3b4d616.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100602.36/warc/f8aeb312b4f46c28a6b9bedf091314045d4500cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/04ac9398069d4cff7fdc199b9d11f8eeb0836395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/1a061eaa9846079a3fcb5ee49143aeb1b3d684c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/1af9ab6b7eb59647f9ee930391ae045efc023e94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/4b4cc305a7ebe71fc33b2c22ba82c8cde431eb2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/602fc710082b34cfea778b8906f9e2096e16d91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/917b7ebf19ad19b60e3d0df6ab4aa681f2f91b56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/cc147a82ef49c9fad49314862fdc79148a7091b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/e5a7f8b7d90a2cab6efe24cfbbf40d5cffbf0d09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100603.33/warc/ea98b1d92d5fb8774a11560dc52496eb9777839c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/10e5eabdf1d1912cc88c4109a9922a7c8d69856f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/1ef50419e008f2ca4d4566ced8b7f8becf8316f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/208f7e99fbeba1ca192bd904f85fb8869db59bc3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/5a813d07e1d3d4582980b95487ee2ab06ba980e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/6a1596e031db5b6b952c764932df536f2b16ef2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/a4bbcec139292e5237098d8feefd311c0d33c8e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/b41613f2e73cadb148473d3a42a68a3353d817e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/bab80427f77b70c3f71d3c4d62072aecd822b3bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100626.1/warc/f92c1ff80888e4dc589d2c8152e9be7259841aea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/09096521badcd7c033d75a7871eb6023bc9e9b44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/33433d66073487875f794b607a19b35bbd5dd312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/4abb5bf113744378f230ef9145fd0d4fce635a49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/505e8c92b7d00f1505debe68e296064032d0fe74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/5d51fa69af96000dcd91e3efc11b154e2bedefaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/707c7ea70c695ed5084f4759352c4f075cd1c731.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/7b6f59ab554b4481740782c7cc46563101dd34da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/b43ccccf0d49d3e8635a72fd61a09ee72f6eb032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100632.0/warc/b959945fa126af5d94cb1b9154d3f78dc4fd078b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/1af7f6fe259f43dc4f30fdc57b2d533fe4c53061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/2f828555515360727f95cda9827a41031949bb15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/2fdff647252642ff08ba4698eb2a93298f4544bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/3cbd77b49e5c18af7b0d4bdbbf35d512a3cf8f26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/4419dfa6a7762305f248f0c378387f68faafb681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/4f21347172e95847dba6eceb0760eaf318c550fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/6abfb37c4a2f34155d901dd78591ad61cba712ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/743f8111433f89e872587caf5add3317496f2fcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100650.21/warc/a3c19448aee0b190f5daa25fa24d048338ec1799.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/19338aa0b2e1cea71a964271c38a815e1a4d92a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/1b815746569a35941b0102558f356cbf7120a0f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/7bb4facda5bca00e82fb601472cb8a867973e11f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/7df035aef7dedb04382e50d2d13e8623ff206112.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/b48e29ce448f62181138729df920df59d7c87b4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/c6a6668bd016a94d3df90102e56263be4c422fe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/c7f5cb4338f4da11579502e3685029cd0dc61faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/dffae131e985f0a87c765af4051112666dd076a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100651.34/warc/f465818c3b193bec53560d9315bd35aad655b0ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/1f449771d9c2cff51f6cc2963ad9ef666cea7c6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/355e0d107bff5151cffe0d9045ebece216f48901.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/48a2647b0f952fee7aec2072205f4dfe00b8bc83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/65d91f17a07c56272a4cd3af57834d6fb815dfdb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/6e6dba9f9828b9c56f2fbf490a20703e23b832ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/70c8ed0fae5f7421aaf3468f96be3407cce0c0a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/90f4ff2c2a333ab4ba5a9755b845d712092ec14b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/9e62210c802f88549fe7e9954eb99c658fd8bc04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100674.56/warc/fcaeeffc7b20006e08cf70eae1352eabfe916856.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/024b445880a4249cf84f74f8ab1a9bf583db6b98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/23fb0ead329f6a9e28cf1acae24cf24642b97739.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/40194f87c52f008fd250db2993dd3a6565ba2599.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/42f556dc1749b7877037af3272e9920a9a50d39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/47cf663057ecf24a1dd905de6fc34afc19e20830.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/4f50ce29a01e0b1418218734b33e3134add92c91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/526342e46e0b06d7e8bf6cb948941a3338b12750.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/5a4f2d563358fedde9929ebbdf5a68346ba94493.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100677.45/warc/5dbf28c8a3cf50216086abde723e745d505fd65b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/29c1cb67989a93bc1cc12902fa23db7c3e85646e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/40baaca935908d0bd5f44ed8905faa208d06dbe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/55cb39eff8db1eee132c563c77592aafc350caf6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/5fe058d3b5e5097d7c6dd2b2edaa078773607c16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/79ed880a28d705899ce6d4bb3793062c1b441256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/bd4594ad5ef9d614ab3f0b9e151164b4b43d0ba0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/d1d6cc498ebabff820334b64c46638ac7dfc3258.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/eb24df13b297d9132a56a27fb59fb739608d4942.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100686.78/warc/f6e1857dcb34884211ba91162019ae99302f3124.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/0274a0f32748c779c73aac5dbf757121cbc6dbda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/2684b078d48cb18a88dd0298e16e760845532ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/2ec029b91359764f7135e4391044aaf7950937cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/388ae5cdcbe58c521b7d386563073dd193e267b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/4d2015016d24877235b773f40eec734624c4eaff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/5a8b31a4ac3b5425d2b7b407558c04c84ac1e509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/72dc273752dd918fbfda32cc0352f6f90509e6ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/b2ed50e262362899f44b6bd05b67c68715800939.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100705.19/warc/eb2c7a49ac737c526201b749bc3c478917aa8423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/025c7344e5c18a71309aa9784f45551491fe2ecf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/25863b02f6ed757551347eec6809797fb4851079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/559578ac0b34e1ab7fd3e88b2a4443af81157dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/90503d4bae4510d5effd06b1c533e0928094e5af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/b430f5ffcbea77bace187c74bb8c993a207cdf15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/c934dbc1c51fac34246b5b8ab7974027150073da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/d407e9fd937bacd850d0b850c88674babb00f449.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/ed3070f6991557e537dfb1302980b5eff2b10b4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100710.22/warc/fee5169703b7ae6adcc193f889765254dea5b2c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/264b8ca2f5a7b1181aaa050aa6b23b80d1bab010.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/3fc8547a6e17f4cc93d0e943c364aaa15e28a084.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/8c4cd7b9207a26207bf34fb807b63f5891026594.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/9e3edcfa143091d7bbf8f2f7477a3e8371228143.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/a61b15b7cbeeafa9ba2f128c3ac808fe63577b61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/ad0256ae626e92cf15e4a82b3a4c146b205289de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/d820d2674d7dba3cee4133f64b2e5deb327b380b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/ef31669394d4acd5bbf8eefaa48b416d351117ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100724.48/warc/f011c678ea059bac6fa7196357553da1a929601c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/3b6ea220feecd45d681fc2f1e20e1089d3e9d8db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/5f616083c8850def853c4a680884e24dad69c146.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/85de165dbc5532521a7335c319eddd4937440634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/9beafc739af9bce3c8e4cebec8a953afbe19c9b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/b2998f8588d001f579c2ff24eea9416e53a23dac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/b2af14415db624941a924a0581233127a5091fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/da3cc1e1c150fe2e1523801064a00cd0297faae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/de37a6e23ecd7f3380132b3aff46bac765338509.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100739.50/warc/ea422bcd7a5b42885c9443fe603f51e744db86a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/8a737328d8df5f6505d03206feaa63b9a5d4006a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/97a2f4d8aec558058b67554f7af4bce30dad4a92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/a715ef487169746bcceadfdd9ab027240bdef153.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/bdf22aa9f7f8a0593c587ae8acb99faa4477ebfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/c32dd1388e0d660d40050ddb80b2220f0f6c411d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/dc8379fe78637695d20db9f1781e2e92bb3942e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/e679b66c4222f4c267fb03448a88965715438d33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/eb53ecdf0ca069bc6ff7c1110c2c0f8c3e466ddf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100745.32/warc/fec91e8262c1d96a1e24cc41052f05c4508e23cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/075d541382008ce4844329ef526009f87dae0aab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/0b33b51bd6e176ec44f32878e11d796dc68abd12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/393628c2945b5747d77bb1d936a34f08efa72ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/67877e673d62454626aeb2d750dd0358b578f9c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/bee75f9bb8e597b76a0e891d8e3eb5a2d163b55a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/dbd596577f304feff9f8b532a6125c8346e2f49b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/dce4445d3ec222cf0567d11a594ca14a23163805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/ecd9286bf383e0306e433f5edf531dfb367b90f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100762.64/warc/fa40a06fb1c3202de4451dfc1cb4b722c9067ad5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/22e442f0165b268c735dd2bdde7d8b3398cdf177.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/3233384099797df8845b9af8e69b7a3043248043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/3ad2305b9f7533b130fad125d7d89174e2d7f9a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/525ce85b073f88aed3dc5954e01855e16d99db06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/68c50bf5679a3142a0bfcb79a2003828d7160c1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/8de081ef8522741ef3f8871197716d7f06176f5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/bb68b896744ca49f1ab8e513eacc19b36d5d039b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/be99890a711e14a15b9af48d02ca1ec3c2d322cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100769.54/warc/f7a363155a35c930ba6f682d236b571d916defa2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/06956702631ebef1cf487ba5a3b17d9e116fa264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/3a0aaa9490ac6f0a2480d8e2d17b4efab02bfad0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/52ee3dc4013c4ede297d783f8230ac672b437f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/6ad0789e089eff36c11530e561357396c550def3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/76a3a9fa8af9e9346fc2858a91d5548198544ce8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/98fc857ae1ee05c98ef089e0e21e38ed13bbc34b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/9ac9b679e8c7da3a04de87e95c0ff75667c6edbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/b52270b628141d7c8dc3a26817b5c6fc940d40c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100779.51/warc/e5fd8755395e7f2c92b154ffafdb6fe15c39d061.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/103376e1a0d22fe48862039849021fcfd89688ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/26880aa981ecd293da8c6639e7e34bd7aff362a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/2fed686dc3bd647e0b04535d39c2cc9d21c4ddf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/4e3a17efd916fabc27f8483025d8b49d3a637762.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/5a2215a474b401b28e2c272a590da931cef37013.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/6b21f98607e9f87515d2ac11e5892c49905942b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/993535557ebe012fd3388ca163ddaf3aa3c9171b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/b55c0c35e7e041ad1bb2557c9e0ca6adb2d68d01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100781.60/warc/c840a83ea5ef0d56290bea30aa49bae2a7f51a1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/4b572bb864e28cbe00092a85e6e0233a99e0c6fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/4e42e35b18f0a5bbb3b220875d8a80e1ed39b417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/5b87859fcb9517c98b17e49f3954041a84484f69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/6177a15cedd4c5c73585cf0b22439be20299c445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/807194b8af6bc859c7dd703d01d79140e8906d9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/869686e83ea00c8e5e6f0beecccddc32b8b00e54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/92912b4a6b100e0c39d0ddeaf76e0bdb9987b31f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/9e7de07c4706a83ab112b2a8bb04a82d07a9c97e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100800.25/warc/ba2f01f90a181c354ee2ee1639828d2688b5dcda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/110643f6eb5a1d6f7b52c688d066cdd5d70c5767.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/1b110d7b910cd7bee0932d7f6c336ae8315eb472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/2571333d1d1a17522637f76061963b10808ac0f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/521e834ce3ebd2d84aa70fccdb2236770b3d26c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/98c0a99986ec99374469c2173d53e3138b6b7a6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/a4d2b4e5897278f7a2c061b44fcc058e88b4e187.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/aba3965abf6c33c70f30749af57f7bb3dc215f0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/c98e00acf270ba0ce7bfe09ee81d6dd6ebc7a344.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100873.6/warc/faa1166f600c82df60fa79a5de4d05903fdf3e2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/16791824bf5b24ebd2561f0086d6b51b1f35c9ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/3a47199b7659607f7bbebe2b8f83941e3dfcd9b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/5e9485a11f86df621e7a9171a269a1f4482777af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/8587b6b494528c2efad80bb58563ccf73603f172.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/8c447d081a2d14dcb7091536aa1f04ea98f6d8ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/9a113cb41bd394cdc5753134372957d2ec9ea9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/9cf07b3d3189fc2d367e78bca2a13700a0d01f66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/ce62d367c12603c9ac8f15c48e803bfcae677b9c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100909.82/warc/e516d75d122600b9eb7029737a07badd93f29e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/1463d07554bfbba2dcaaf363c525bfb5c09199d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/29e0fee0eb776cf17c15fff5bd08aeeb25d13fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/2b0cbb4cd2d946d0d7c992af67d2275557516e05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/2eed0c7d44579963c57948b30fc5936e27eee916.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/33774ceb040daa01ef52faa8e14fa54afd9dd733.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/41acc35ab5d6ca45414c175cc86eca8a9df6f772.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/47fc5c2a07186aaff02db4bec565013efcbb7571.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/70919ffb5392466b0a3fb4daa3987d36458b0cf4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100912.91/warc/bb018ac903a2d043fe3a9a5bd70ef7a8fe9f8f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/345b98f2c704a5642d866ee73b8ba6af5df8bbc6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/408a020830aaac5ffb311cb20b61e94d3eb0d57e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/7d9f228e53f6b9af56b9282d21945f278dae9597.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/8b2443fb58f4922ac8c99432a61f26c031df0e44.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/8e0a6c3dfb182d2c2dc89369fd820bc20f8d5239.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/8fadc42a3305545f8d54db46ccfe10d0a960442a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/9d508d47ee24e81b452d4d56d34aa5158e518549.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/b1e5b67f7dd66ca4a990dcfe0aa4d57937affe8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100942.92/warc/e4509168ef7da983bc944e9f6ad675620f2e33d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/3c93594923a10d3ec902a5ef86cce31b4338d2dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/53506bce7276e112f81f8ee7a906a11654bec10e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/5b380ec605fd5c35fe5269204f43bf4496fd3de4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/671d75e8649342b455053f6ed2e07ef5d92cbbca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/90ade5a7ed8da904e6a0c74b4ad63fea2ec04634.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/bd309aea18f3d85edc6ee0ce568dd8d638ca3746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/c56066aca8756ca23d18d552d644f8056c1e6b62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/ef456603aca1159de27115391303a682db3548df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100972.58/warc/f1c802fab895d9dc73fcc2b1dbc264b307c074fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/372e9cfa4623d3ef2cce491b0b6c8185f04b58de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/3fd42de4663c575b1a3442641fb32e7a080ef197.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/4e76459c741d37fc1ded5ee1172e183427087483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/61030bfd9b1af6774a7b337f108c0396de348ad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/6b0064b4e8a913a8a06b912449be759a430ccc74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/71f2f809a12f549237d2786eff25b8d6cc438b3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/9ca8e30c141440ac5fe2352be4532d96d6d177db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/d9df461b3140a80fab7ef06806f234a9ff1ca7a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679100989.75/warc/eda1a5e4a8c3dca0a151b3ed1c02785513feb5f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/2004818d6fdf59681853c5480ba47f77555c0eca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/297e08e968e6e95661678c4b146dca1e800632ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/2f8e873fb03714efe48b4869e28f18524d84e71f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/38837bd66df27b38e3120bbc0fb036a52a6e0651.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/4ab87c754c789bb37d51fd9f57922a52ac6be07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/aa47014e144ac5752dbec75e829ad9080982d4d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/c1788152899574751f7288242931eb942a1b112b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/e6b653d3511ac79c98b5bdf442b32d52f1246a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101195.85/warc/eb9dd004a33a114907f889b5b560745fbbefef75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/03fc1eb567ead9d846c82800b16cb445b68ff41e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/4b03692738a61fded19427f71f61a7bafedcd27c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/562a01ed2ff9857b0d425fb54e7645d46d0e31ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/578072c47c17dcd1f1815d0eaccdc8d15efdb194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/58cd63c7503aae56c95267f1d8bf7ba469d332af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/842469a9941bf175b1f51c979a3aa462a3544805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/a91c3158f14e5fd33ce6306b1e08757fbd668c89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/d5ee458a756944b77fe8e063a72ff4de11286640.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101282.74/warc/eb6f393fdb5961fe3ffbf9a1e1474820c3875176.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/2a9dc61d3e2af58f8354c1a2eaa50873828a8891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/3af4119aac6cab1a00362af2857dfc47f3d273a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/5ed36f2fa3546e55cb54ce5476e4af1fc938fdf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/6d0a61742c85e5061456f8120d8619b24004c543.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/78548e56336d351f30192dbed772626051f86945.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/92e0c59513a046b4d9dfaeb2ebcd52edb4043023.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/94a893024cd1b73992be0d502cd9ce0557cc25b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/b1adc6670dfbe6c56e2494efed5fce8716e4769a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679101779.95/warc/fe83f6ab07625e7328dc6484239688b70dd6d47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/18d4ac886cd4344b138b06ffea70bf1e0ca1b5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/3b439eea17b200d684e5d68c1fdcafd1fa4eeebe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/43298d5ba8542c88dfb6e29bc04fb029e18673f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/44c12aa6ade4bfc16d308268bc93ba15796f7000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/452819037bc2919f3a23ea7b8e9962f9985e8341.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/85fa12f97660146d4b4ea4c4c41c57bbf553bbea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/d97bd429c529c9eb655930197bb35ac432a1009b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/e2d5d29af0c14689acfa87164942846cf4c913c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102469.83/warc/ecaafd27f43fbbcec83c2f6c8d4cbf69ee0ca391.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/14c0353661802691499adea4153603d25de56b1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/155bea5aaf771caef2fdb798f461a2998e3df144.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/18b5f8436ad87c0b2c019f5c8c2e36d93b09c540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/1964f1b26a9a25cadb4f59c50d300ff96717812a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/4169cde3f0cebf9d8a974a2171ebe259fe07522b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/5c2bd69c2c3a83bf71fed9245b21ed666b4b75cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/6f6e1c8c1acb56738675dd2099abbdeca0d75302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/90719d5e603142c5e88b7e6e7c66aedeb64d75a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102612.80/warc/c33df80833a222d5191d0d1f5debbba71dcd1e6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/76bc89ce05e9f4134634ffd7d0109453b609eafc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/7fe214686f9335bd44ce7b8d1eee3bf79fb724cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/8674c9f4f91ee1b01f0794610048d75c3c02f33a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/c39ecb09dab6b3ee6de08352f146693425ed1c68.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/dec89d5be79083dae1a11265f61971c1d7b2c98b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/e7dbc9e08fe591f6b5162f43b354c364354877e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/f28e76da679a94f38dd0edfa37f80a71e04f7507.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/f39a9ac1b9a2eb1394ab8f00ee45f064bf946dbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102637.84/warc/f7e8087ede4f2923d239b5dd6940aa4e0ab23082.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/0754faab5d8bd90a6d3e3199b3c367f1b506f705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/0a30c5d6bcf7acbb13641f4dc0d23cf34dbe43dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/632e9a019885dad36fe659d3c6eec04c48c0f4ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/669f7531ba879496704077a88dbb6661a624ce8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/6ee0143a2d5d28f5ef54e82c294998aeec0b48cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/70fc1f23cf3b02c38206d651c64c635e188f12df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/c545fcc70c9a5f4d257844c47d9c160710f2f3f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/e426281532bd87c12f7308188ebebbc5d53f84f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679102697.89/warc/e7167d86af426cf1f221fdfcbfa802b9d4e86889.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/09c7b1f94e56734e2375307e94a7fb5173b96563.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/34d61e019483d120a5cb5aa874a949e7d8cec3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/4fd787352359d3656197d38fc6fd9b1f3f6f142b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/52a6cbd0ec51eacff859ad8a78ff3dec6e6c78bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/7e51d5967b88af8f888c8d1f20b6fc54b6aba740.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/92768a8d757c8b0bb2c389725bb3a8a0505b507f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/9782f091455e619da2cf5abf4c5affe4c7271b64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/b7b560d51208e9db461a0849e7ce52fe166d4768.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103464.86/warc/c67121862aae9c7862fb69920256fbafae33c619.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/174c821f8ffa2eb25a234817de7c931f2558ac7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/30a726a067a7356b845c1d9f17e5295977aec33a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/328035418de7cd42ded76ec4bad2dcf4fd38d48e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/4edf82c3fe76be53ee75b6f7b2bb1080925d9f3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/564c7cc55c39b44fe34dc3ed6123e513de501acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/939f2b0ff8a8d1c1d0505bf3e13429ab0df98744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/af0b506f57497f9c03248f56dc8eb1d57acc53d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/b226d1ae0894187c93b88e0c19cebd734dfcc8b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103558.93/warc/d2ff06430e4fb601b446ad31e69d7bdb2607698e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/19b88981b3129824144b9368441913529d58b06b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/22d23f76198ff5732c2c992502c17f7054f82a2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/29cce375a2587f72ec58801875772db343a5b043.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/3e27e3b87433351d9ea1a6a2d231219feadf075a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/55c20562a2de95d8f8d144080a0085aab07e567d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/585e2953bc9c4c8c3101d1b6f1eee95eea23c0dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/a3f15dd6ee3f9995f76330af28545dca5a8703a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/afd5d91f1cd195796c13cbe51d0fd6fb3166a3ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679103810.88/warc/d9a3e18a77820c24580825f9943c1457941a09d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/076b89a132e790fbca0f198ef53fa25e0c8a4c5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/088c6d0894ccf4bd72f7eff8c79e7d097b7f7405.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/2f59f4ade6a7d3dd2e053dd9501ed0d184aad725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/66416853205e69f77c584fa0c7ed814f918a77a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/78d20c6dda427095122fbbd19fbab06afe881915.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/8797d108ed2dcb81f77b5f8c2c37072b128b13fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/ad4ea527c95f8268f0cf178a58daf3d49766c1dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/b693d51c38249bec30d0987adc9af764d25d9708.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679511159.96/warc/e5013eaa2c648e2ceae9d30cb5ad0c054d4f6813.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/07b6ee539642bad3e777d5c56c59534a7a2a7a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/317503e6a6401150e005ad402593a0210d9aa6ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/3a90bddf49d14b759ade64b0b374d23647035d0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/790892bf391c74ea71cc610ec113dd281f8078cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/7ae5902a9c51fc3ac9f1fc4f096f507d0efc7144.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/7d2e930c5a30425cd684a1611a148116b08e1c0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/a2a00b568bc75fa5b803a1736d4666ed21700863.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/e914899cf609c3a3ac8debd3729304da62477bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679515260.97/warc/f83ae011b1ea9a9dde20a3658a010c5c56661f81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/1850207cedcc9443ef60ec04d7dc0da07a3c1fcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/196072eb5f07d5bb6bc06af3d5086a3a398d6bda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/23e86f37f8a4ce70ea5ef816a2d1547751d4d2b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/25c270861195cee0259b500a65879669b08f61f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/27a46d603a0a16eed6b68215cf4594f8df36cf1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/5f74c69a28f13e1e06b32d86f3c6620ecda2e797.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/797b0787276e1ef01057119e1d53d64e48ffec89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/99d45cfcf8c4f2c696281aba190db2df7d0664a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679516047.98/warc/9f9b2e0f0d31d4c3f7e1d446a88a7e227474c687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/43158d24e1c26b4a53c62faaf20697172eb265eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/45477eaa111b7f3b1741cb787699b88eb4a55b37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/45f6ecfb9115e8a6bfcb5aa262691d222439de5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/6ec73ec4c7cbc74a2ccc4002bb8fec9304e980ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/85d6b3c0efd057c231b5637be7f0e0827eecf7ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/b2f2fcc74c2d59a02cd94439ecb6e24ceaafabae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/c2529b05f1aa6dea496fac31996b0c15c641b654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/dbba008e7a3eaeb31f51a3b6377f13268102ab08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/1700679518883.99/warc/e1b7f364819cf13ed1c205386e7ec48975ac6e16.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2023-50 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2023-50/mix/input" + output: "/tmp/cccc-CC-MAIN-2023-50/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2024-10.yaml b/configs/cccc/mixer/CC-MAIN-2024-10.yaml new file mode 100644 index 00000000..3c0ffbfa --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2024-10.yaml @@ -0,0 +1,1014 @@ + +streams: + - name: cccc-CC-MAIN-2024-10 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/16601d1c39e9186eed835b109b5146f326b1510d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/1e931e2b3cc313555129fa4fce84d1d3e355713e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/3cc2b53e6e96c068e924070d6522b1b976ee9ace.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/53abf26a3fdf4903f0bbeaccd49d06aeab1108f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/979c5f44e56f1b9007e4c1b9f5b5ba9fab86a0d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/990baa29c50f47a497ccb80940b64d67a188d39e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/9cd1976e3fc1d44e3d156c0d71f3f706419693f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/b60f836f5861832fde0debf8ad6ac726da6ecc74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473347.0/warc/f954f00c9dc0a24777fc0aa8dffdf884d9750f0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/2061e0d19b9ac3510863827d9782ba6d92ea4707.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/4acd4775b1f1bc2b9f33466e898c84f772f72595.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/4d8f1890f79a6c4e9abd5fad7ffff63d2d251e3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/4e097d3fa73bf3a40becef0de2425129643cf378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/4f4cfd71e0200ca05424521ba003478fab37ab1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/8351de2c89ec26d11882d8621b3603b832a93b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/9c55a889dbef2e311a1dc0644b5610a954bbc906.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/cb4cd4bf8df51f40859fc187cdcd4fa28a7509cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473360.9/warc/ed213484a597d5c7961fb2858142a11dab98d5f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/1209e3733dfeeabf2c1d11bdbc87b3ba9646fe0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/1f0cefd81a9c802b7c0fbb5bb75ed08f4a8fbca6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/411f497344bc45783fa2b0752086c7039922511a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/6970bef36a56efaa767649248a7535e19167ca45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/9338334045e594f1c79cec62de080d2d94340cc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/ab0915bb9a14e0284e6807655d48e9d0943b3924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/bf37dbdb785fc9fe07496c6ed897552847aed269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/c5f7e80282c248318d9b05c2c59fbe864e61bae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473370.18/warc/ef2af10bca7588032dcf7d7cfbd8c35d9aab494e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/2371a43cdba3d23a067248f2dbe16867a5328131.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/3acb34ac7c0fe893d0e0513e5244eb1a9631dce5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/3e10ea3317867a6ecfea7e1c82bda49974aad540.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/46d68a22800fdedc78670bb373fb2c7a5785dfb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/63c36087a4d20133519936734d7d19dca65ec9d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/75507ac386994f649d0b96cc976a9ee4700dbf1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/86c867f4de6333650542c249333cd553325624ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/c3707864b89f4bf364aa9eb7dbdf7fd7708bbd60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473401.5/warc/cbbb3cb975f79e88bd531ecdf7093edd0a1c3f79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/045fb795326f2880c3b042984d65fb518be4c2b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/175a3e4f97f4b2834a7c4e7e026c5244bf19c1d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/3a4146c0f45919971358061bb74074adc6e7700f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/59fa46114a3858db227ff98b6abd95c5f0688f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/6fc99f5f285f8a213986602cd1b63cc3750e26b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/7e79695e5b2fadd71eb42e768e61229393028a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/8ad721b17b1e7437648801b3cb793a0ccb8231be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/8c85e5dc18eee76aa68a27fa8fda0db03b68bead.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473472.21/warc/b8b031ebacd47c7f6a48d0d9d353f01ef8acf97d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/04940c86d07091c67a48dbf13058ce59cad37586.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/0505460d9abc951ec391177d57bd1312ca2a7e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/15dd17ec21dbe796350abbff476c9b2d61d41121.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/33be255a765ecc89c01bc27941518e67fa16e87d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/a1c7078fb3ab34a7b2d2cd71872ac2f8569fb43d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/caf0230cc2f6072a9be1d15633c13679209f06e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/cde62dd45907c2243f1a184fd8b5b9d7a0e3f855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/df49745754c8efd4d647f51264dbdcfcab2a2a08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473518.6/warc/e2f6b2d2432205566994a706061614155bac292e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/0565f41fb7d7113883d97afb4a10b50bf246f1ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/13b6b8af88fa13b912db766ec22097cb296a240c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/2baf11f35ea80c98d4bbeed6d13cab98a27dc923.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/2f0106e62b135e386fced1faaac2e767cbbcc11b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/4507126b7c40203b88a5af1f82d85324c2f32b09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/6b1484e94ee67ee4fdaf9e2328feeee64c4bb50e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/acc880fb547907925f1f8197abd9f4d0a3f9788b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/d4741cbf7eb8701b5effd2509310a58d52dfd91b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473524.88/warc/ffa3856770ddf04a985e165f1e46d8a93833ab57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/0a3ae16e24ad20fe4bff260f5e07bb6201f43365.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/0dc18756e323d8ad4623ab5292e7f92dded1a07e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/31d484eb208f854aa4d1459a98b017fe09ba12fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/3d0759efecfe13d94021efac40e39c54d004358d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/4972bbba3c246d9f712fb93f0d16b62963ee1568.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/5a01d0d88391b2e5e93e8638247bd157460ae7c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/6e550b2657ab83f047197c42986ee063b7359db5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/9a9c4d863ba24fafcaf2661fcdb9d94f721aee6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473558.16/warc/d5f4e07c47c21fb42d76b82297c7f8517cd401c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/035a0aaac701d404ac6245731d1fc44d90378f8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/1b00670cd2a774d24d2f943bc1fa6a4707cb33c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/3fe4f1ae327371acfb4c22646cd17885be455e62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/4d486f7b1ea3132d2a75725a1950bb832f286318.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/4defa61e43a41f73448c08df82aa348028b715f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/65f7b8d8b61400e6575122c00831380caa5130f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/7563fb563347277eb1599b25db019694d221aa90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/f4d5cf7db3390944bd77e63ee819b620006fe83e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473598.4/warc/fcef1e5473f2aa71e4a696b45b1c9c030f984f76.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/1d5bdc838740629d987f0ab027a1286c26c86920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/1db7294652ad51999eb62c37321d2684fd9b8da3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/25155e6cdc4991854bb66226b5a2d0016527ac96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/3c7a3d5f32eb276611a73ecd80be5079b497edb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/3cc0fbe14f2ff7ecfaee877609783a3f6563f9e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/52a84e64c42a514985813d4d5058b6cc20f612e3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/cc89cec64c661de7a6c0b28f7d7de6095140d48f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/d18db82afdce70f396c05db84745f99ef0a47831.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473690.28/warc/e42d14a267f5cf1d3f75b1d5ce1170e986a9ed74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/055198c7f11d1e8a4d52a75dd3d0224dc2ab5190.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/1b98c04259660970e072c665c20a2e44cf3207f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/30ebd079dbd50476e5e28c8c46961581fc113010.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/32a48891895ce70c13835e2872a4fad8f9a544ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/4366226e5485ac5c998df1ddf14add373cb757d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/4eb876bd3decc1ab5df856995c976a575e6e4e2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/562ad21a61fabde799d06bcb8d160798317b30a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/5bfb3f3f2296ac734ac9c3855f038841b8486666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473735.7/warc/cecae7d13f417c6d5d0475ac4c8838492f2be535.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/1be847109766b5e9b2e3454e510074ae6d368b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/3f8736d2f6f7ce2576ae4ec958dae4f136236bc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/8a51faec24b00f29cd0509feba7d026e0a4c0771.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/8ff821aaca9c2aa202c729e6e851c081e6fdafd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/a25edb3f6f26e667a28c541d55b2bbbd46f42893.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/c76e736b7bf928f074030fa3af9fbc19e46ecc8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/db3d90c6c94349b065242f2bd0bcab5e57a619e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/f301338b852cf4c8f36ae43fbaf98f4b58f1c687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473738.92/warc/f8763a536bbc6084bdd4a9da76716b1af2eb2953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/0671cad13e1701cf3705341d7ec5631f2062d792.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/0d0b42cee520043731bcb6c1ba6e49a7535ccab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/2013eb7ec1feb4f4faac6873005957701ed32606.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/33cd5a1e1d79cc897023073e6efe0e01f2478f15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/6858d1b0a8bbc9b4ae23ece32f3428e1e4fadfe5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/a9cc88fbbb7a74405ab5c52c8a559aa09be3ceec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/c6366ea3f0fb8bb5044732629577a2211d48e5ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/d1ea7c918cfe4939e33974838ef34a10615d8523.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473819.62/warc/d5e113811dd35aedd5ed68198b88bbcbd7c86244.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/0b6b69197822a7e1e070ba5ab246b1500545c687.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/4ebef873bb3ead0a2b73b0b0a13785bf43beba69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/61fe0119522a76e535b6d6b69b9b60e3a93633c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/6327f4976738e5886a04cc4f71ecf955f43a8817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/650057e4fea8484ab223509915852414df548a04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/7a2968541f4a6ff084bef477e997298be22c8373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/a73e9dd457a2d576eef52954546658bf7c45a6a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/d758fb5684321bd9dd9a35c30d774f77894059be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.13/warc/fcc17603fa50055d803c44a2a04bd0e6a02cfe7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/18901e1b98b4164de7d50060638506685783a532.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/20cc5837120d45b43ea39510064b7c67fe47f091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/262b235437cc220b7acf497080bd8c1ddf7efe2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/42e95d9f7031d573901239013f3073524330a89e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/6edae1bc225c0346d9fa4030dbce1a7531dd6307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/7b5ac75514fcaec56c6e2a3892c073f7ba5ac14a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/8c8891ddee360dbcd1b08f10aa898c73f09da5b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/ca960ed76e010cb97c1a25c6588a5d2022ab5090.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473824.45/warc/f4ac6ed8934c1bc27c240e71545a33f46c7d4416.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/087a6aa9d69b13f34ba8753ca8709138ce8370a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/1d76df83cf771f33f4065935ea29a9089443afe4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/22bfa16ba0099cf2c6c40fc4ccbf9d3e446a8a95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/3f7acedf505a7e924ebc8a39188b46d6989184f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/49215cf3ed82623c28504d9206eab96dca562a56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/7baa25750f4e7d3afb43f8f56ee4cb303140254a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/8542db5bc31d180db2b4dec8d062903dedf7fb21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/8dc9541b1db655ff2dfd678641eee2e88a8e8a05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947473871.23/warc/d79095cbda2e4f535bd9ce3da66b728c484edcd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/0a5baca854e2f4997342df09354a2a45e2e0b6a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/1bbebb2613e628261e03e588264a430e3949006c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/4653ad5660fb821bbe90f599fc55b855024720e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/602f1a20a837e6f53678f88d5139ba1d6771707a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/662241a818e31092a2fa1d514ca65daceedaf445.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/85caf7c5937d96ef91a6dcdfe6f53eb5132467ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/a8cf7a29d919ffec26b6d46093fd2c7564855bb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/c016d4a6eff408b0744e57e6575763677786188c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474360.86/warc/eb8081de478aa7002421a3253c76a06baac2138e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/020662834eb032a15b60be8d8cf536708e30ee9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/0560a92f97c011973acf25c5d7ba4977f27ad07b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/21bdb71363845d43fb0301decd2f44b67b778c27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/2f18982bb0b9ad22eb50a03ca270aebb06371d86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/ba2efc8b5e5d362147d0133b08c4c060a3c4d977.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/cafc386a1203cf560255e4f82a7d22e10630627f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/cdc5809e64cb83c64e008a23d6abfe9990ca473c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/ec2473a346c4d4be29b34e418f94a2280b8f3764.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474361.75/warc/f1f564355cc495cd979ea4dfa08bdcb1e581b098.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/082815727269a369c8c5e0a5e9f81add2b195bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/432872ee1838a88163538c577a9574ea72d26c90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/4793cad5a7378a128a58b6e0a9555ac6c8b85f94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/71d1d89829b5e4d83b43adfccb21bb342d53a2b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/992b3570edcdb179ad6ced492086fc0085c36e82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/c61e9c6df1afafc1d3800b49de8a2b86d5c19969.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/f6812b439dcf46d7881ba346b7151844f5ee387f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/f969e63ffb5a1e63b136f5dbe18c1c256d2af0c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474377.60/warc/fc06b4ad6a0df5dd3e2fde8450d3eaf8d543551f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/00e065d249271f0b4d9c6b19bdb012d23736cba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/10509e0157e5f85ad8415c5ded197d67de6562de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/21a71dc8094e8039d0094e48dc6f9a3ab2d92fbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/4edbabc57447aa546227816797adf615d0ece9ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/513e71ce1ae0295639f700aba5e74ee573868f53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/850ce06e25d242333c124aea5830cd8fefa0b8d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/9f0167e8a30730464a3bc5f8c15721658b228148.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/d73b8daa4cfebee88f69b7d9448264bf3d9f3bcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474412.46/warc/e73b5477c468daff241b4406fd2f723247c7b6ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/08638f2f8c64ad16294a4fafbe6ef8d8ae50afca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/1ffc3236c19a03b72146dfe705d72ba7107167e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/4c15db96ec2e86f68de6db4b8e94ed2c2c658b3f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/c370821f7a7a7e7776e9fb27a68c26a926c8dfd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/d251f6b25259bdce62994de7640a2716890e2a34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/decd359c4abbd4352435e61712fde1ae1d41a91c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/e18aff9408e28dfcd005866d7751d34e61c57bd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/eee7a578bd8623d4d338cb5f48adf96f4e9e42af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474440.42/warc/f0dd65b90442b394b83d3a9704932c17235e71dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/63b503ebfbd0de4d2e91c10f4cb559409848515a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/8f44174f1e66cf654f77d6c60cd007178c37d3f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/994bfe4b650307f3b4bb4b9243a1e6b5b3832506.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/9b8fbcf83509b1269725def3e8de7b24ced40f46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/c40014c1e89496f774f131a3b810c93638fde3e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/e6e487d44e374e2bb3f0e2c891349bab8eba1a23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/fa3fdc05043bb9b458c226392b75bdc3b6a943b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/fa4c5d08ca319d199588b87d33c0be3d6198eeac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474445.77/warc/faadde48eafbdf8b59dc003c34f5514b445892ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/160b603873b663fde274233ccf43d3268ab8c8d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/632b54a49388573b8b7a3cca53bf0d9b2cfae6d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/7613afb175a8ca9b9c28bdc2e37b99defda10cc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/80796cfbd6249f7e38159a308619ba46f8bfcd1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/83119c6e9050c188531de4aaa4d798573f6d269b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/aff0c970a4544663e61c77e8a2dab3f3cb572f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/c3001383808010e50a5f612eae5bacfa6e5d5ee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/c617caad2fb66f860809b0f289f90bde790027c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474470.37/warc/fffc412cc04bc8d614c216dbc8765c8f480fd6f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/4da7363ab40f83dc9fbab527e64aa203e46d7d9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/62168f24cf6e5a52b699a9fcad573f17246f7022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/7a2c1af8a81c9dc2489fb9ee8ce5c901e4e4ce91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/7b2efdd3a34abf8ba0c885728c36dba4c3363a80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/97db6f23a9407684fbf4b58c1e00eeb8b5c5ade7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/9b3c9bd7d34ad22afc61e88808bb6f13f57001e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/a8ffef2ceba52e6a9232af96de1b80212c6f211a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/dc40106c83dc2f315646ffeb5160788c792c37cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474482.98/warc/ffa6be003acd4e56a99011aeb2d9c47b412118dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/02bec34e7713982ec87a972ff20b2268aae05d05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/220936258f626bc58dc988b02b1992b33d66bb42.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/38d5590034c21bd0588b7825abbad970a2806b4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/74a6101623634a77a6e94b99cc3ed6c67a9f627a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/7f8ce4b1e8b573eb97ac0d1f73aad807d0f07647.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/ae444aa88f4702f86f88661116e8e91508e55e79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/ccbaebe5f40053c582445e2e25dca428fcaeac52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/daf8b5fcfec217e991046a14efe83d9567a5e21a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474523.8/warc/dd5792a7a5216f2eb94fc7a74515c84c21120489.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/0842459baeccdbfa1880574bd9f74d485b4f4ec5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/29e8039938fef4bd4ba2c4a96f630dace443bfa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/2cda45b3820c12853adb7e797e376de4f21c1fda.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/534779455bbc63a0a0bfd46e8abd2b6a557f168b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/662fcd808942605c62da37d1711470865d9e3b87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/80e03b067bfa92e740853b8a432e5a9f4d1f5280.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/8d704b7d31a3750032f28ee844dda7b4c9d69f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/a38b3cf0d28b9abe95a2055ec4ef43bee3bb0ac5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474526.76/warc/a787a0c9ef44b2bb80fcf16a19c874e8ed951e10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/082cac7ddadad87714fa83e81cd1537731f6732a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/21b51636f3e1e603fa00f9eac4960af714c1c01e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/778f6f010f6e9f3ad9c0015004fe416e6f5ada95.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/807da568e7c447794e307d26116aa1bf6c7c4f2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/bbaacb5afdef6e1378206641f0af0a9583af7269.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/dc2c01ec2b7004b6d5d50f82bfb51f2013fd28f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/e77037f035b009547d2f467ab5dd85994800d778.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/f295c413f2cea25d4406efb7b4d3e4c939deeaa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474533.12/warc/fe65784b1b5f2757ee885e3327fc4079cfcb18f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/3ce592fffb39ff6abc6f3afefad1446a8d6cae5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/5a4b03ce5057c902918d8a5d877b36c19ae8f35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/6480fa8676a948b6e195c32106dfb5250ef6d4bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/91bf72bd169a2f015dbee4f77fafe3d08ddaeb15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/ab649381e3cb5f3db5ef7dbca2480b342c2b5c8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/bdd5a2787f9fca9981b65f7f1bf3dddf27af0717.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/c32e0a92ab3e951a858c1aec2572bb885d3de3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/efb87d988ebe3cfceb5715e57234621e00ab4b54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474541.96/warc/f1c82261a4807c376b5efc26a9defb74060714de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/4c636965b6145f5316168c177db248df2e2e9805.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/5bb99a6e9833088210ffde60114a4e85795d8934.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/627cb5b67475f96f209db2c5da36932b26aec326.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/b6b9409331310274aa36d9fca993d3c98218e892.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/c0fad2c9837df8d5b6814e9c9132ae78644dfe22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/c7d43241f78747fa24832a01de017101ec4d100e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/dba257663cc628317573c41c45ddbe2cb68c4f2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/e866594487549e16172398fbd751f322b525f710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474544.15/warc/f0d1e06fbef236eefc406deaf6fbdcad89e12373.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/1666dbaa1277aac9272a4fd8ffb1a1b55dd2d233.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/394de2c5809f9c2101ff4201f3e7cf51455dba0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/583cc041fc9d42f428aea8c67dee534a02db7ea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/5b42ef91efd3c32403cff18ecb8ff85e79149103.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/7b27125b83941cc379380f4572ab328c441873a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/7d19291def6db0e4f315350d65c2fc8de907a5fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/9d258cc1c05e1d4119a74a0e7840445253fed3b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/bb3269bcb697e0f0d77621bda877d4a4f50bfb29.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474569.64/warc/d9b6c4f07b108485f29232b20c703fbd27930815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/30d1b95c576baa99dbbf47f297c7c9529fbf6c66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/3fd99953b55dd7d9ddaae47e1dea64becffa8d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/574530a0a99ad8a108006df0d31c9976dfd3ab74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/5d131884dbc9e43ebf41542658cdf4cb0fd3804d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/7596207f65b4c246230d8646010dafdb351e65ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/7f1cae3a95720a3c062dae337319e53d060a5ad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/952f75f9136efb22131fbbdd4ade0d933302ebb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/dbd7e3d1136a6dce529b7c819976d54c5679f3f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474573.20/warc/efa95b9f847408a293d390f2aac0795fa77ae03e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/0e7dad40fc11c8551a07bbcbfcef32453559440e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/1d056ff4285e3ebe1cc0568e0a8de0ba81474292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/32c1988afb76a3fd562f421e37525e124b869e55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/533072fbea305e0a9db1bfc72fc675c33527aed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/7ea6f44c0338cd1c878667e1f3cd28bf711a6cd7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/8cc4be701906a739cb4ced29f0f15ff2a2ed5213.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/8ef86e77bd71c0e48769e53ee5aad9af3a0d3079.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/b132a04cebddcf10a3eb9eb940c59583d3957691.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474581.68/warc/fa2d16a3e9a683c4c997f77b7c5a9bdb43c24f6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/0d4a3afe44ea518be13e9b785494e765fc409b7d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/34fff3bcfbbd21bc6101d03202466a550603229c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/36f276d26ee146a33b252ae28e35362db367e712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/697e0c6d98b41012af8ae28335eb02e57e599e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/6ded35ee3c78dfbd4071e7af57cbbf1eaaae119a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/78b210902eba3b468834aff0a043b32ebf3c4914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/7bfbd34e2b283b581102dc3a93d56d17f8fdbc0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/89c02423ea893f2ebb6f6610c1a1864c7c7150cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474594.56/warc/9e6ef59861a9c239b344d3b18b3cbb7ded6e326c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/264b6eb3350702e03400c9bb01f35370d7aefebc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/5412e77d276636746f3bd919e45d0b34c30720d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/5544889072091bb6fef7f6d069a140e801c0f095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/5f2baaad5280c571c6b36f5918b77c75d6ede289.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/91c637d3580b53843dfe95f0deef98067ab38dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/98fea82cbeac46484f0b8f0422c211bc514ac2b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/d090063da3ff78b5ba0ad509a6e91f866139074b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/d6330903a3d2385fba04a9532bf460557156c26c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474595.59/warc/e74864305d593e36403a2176203e466aeea399af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/01112c633bcdfc37f74b0f55ba27c7dfcd718b88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/1e4b5945cf2cc4660ead6bd5810047a422cadef2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/3117f60f1dcbb3f1646ca8e3eec1cc02780bb28b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/3a28330bf70a4243985702e4812b63b54d30ca78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/5809482abfc50ba2c052b0b400076415fcf9d93c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/6a08d894448f334ffe7fa91370ca789678530c4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/d595a40752e6036a3fb5fa3d029a99582fb5260f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/f77ede3bfb8559d90d6941bc586ab883c033723c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474617.27/warc/f852f8b482945d1a43cc9ea2cfe656561ba145f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/07b1af0056eb3e1b64a2c2d39be0b97650898d2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/1832df5cdc88bbd3d4574ae57039519076e2cdb6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/454e4b7450765d00d00271a9990f1385b6afd3df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/46a502cd9f2c27f39b7dd3c8531ee5c5048c0859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/adb3877961825615cd804f873daa8879a4a83bde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/b77aef1a2bb33c1a6162800b0c362f0b5bd4c852.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/cfe2b73c04765949b8606578416e7100462c3f59.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/da2f475c5a5da369cb56b9f1ee21037186c9e4b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474641.34/warc/ec3a85cabea998a37ede6b8667bd4e88cbbeb796.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/57b395b9d47a5a53991c4713e7b7f86405f3d8f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/686d77cd44a6dd868209e7d4a0cf2ad0e4d11ed5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/9097fdac0f2f94e3ea03bf5b53a708b81e67bfd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/9e0b8ba8f44a5cdfc8b559d23ac635710da7697c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/a099c34478b52bec9701879d52f39dcf25a633b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/afd5f008f6aaa6396e9dbbe18da6a82219b9ee7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/cde70f161099dd31a4f942a4e8cae262f3702c0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/f523d77512e46cd4e3b4bb64a8ca380b1e8440dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474643.29/warc/f665cb5180bae96910d89f0925ce13341258ab22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/5c2f30ffd411ec5fd2531b478a22e346d208981f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/86379868deb1c0384a6534ea86f4228390920862.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/86675e3cccf7a31068be8363b9e385b64b25de87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/9c58f2d9fd1735f3c15f6b53cebf29651df19b2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/9d150f1937f41562f2ba75802dc5741959e7b123.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/b39bd112e8ca44fbfdcc43cfc75a2f59b7bd9a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/d7cf50be695bf2cd1e217f02e89bdfa0b7571661.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/e3f926815bc7cfcf5220b9f6e08fcfe8ff6427f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474649.44/warc/e535dbb721be538aaac19d5d3c3b7dab28d4ad5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/00ef87444400b76c4c1026b6253458228631af30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/112aa1749c73abccd29b3557f4edd488a1be8d6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/3dda566da3ce49de8e8921a27ded21cb5ff0732b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/5a48180659665ccb84bd3f48687bd0a2681207db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/8e718e65b16bf0ecf96666aab22faae304e9d1c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/ad6da347b55124cde8c1588d17da8eec5cc01e98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/b4331aa3cb51d993f43cc7b96bb48c7f636235de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/c8159bb80f0d6ae2eb2f38e856d3924e466284f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474650.85/warc/eefd536c5765c81e81cfe967a9fc7fe138809e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/1c8e4470a55719a0204610da29d606f765145c77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/3a2887f46005e5d61163b23a3e8f782232d721d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/48544f8c9fe1de698741fd7d70718423e60e3ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/66882e4eb2b65abca16c8dc385319deb21b1a965.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/6a042afdadd218e160bbf0969d27114769232f2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/8c9c3609ec5427f41741b1cbdbf602a44e48d111.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/b8da533a7151638f061bea1923bed1290f6a7cfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/e28444192428cd6133d99fa6ef7e860745d9e90b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474653.81/warc/fb1cdca3f1f799fc7642f8699beb21c81d0f48af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/0b0fcf327bd0e38ced3f2136fddfa6d9c4b27b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/1865db6928769ed5a4d7b976dc5cfa6afb2a971d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/2b248ff637d04d1af53f071d4525fee44b111686.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/2d4ed6d14e227fbd8e218f9e9491c018d08e4aeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/b617de77fd8cfbd2077867d0db338513b864d925.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/c187113e1fa87ba09df9fd800dc36e99a7446999.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/e5e9bbd54a5dbb2600e2d178fd1800804e6d8249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/fab88769522de3619e8bd981015339194c82e357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474659.73/warc/fbf2290ca893a1be9c9ea842210d2d0359b23179.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/1a7f22f095b9a190ee27d135f7a91969e9aeeb32.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/56594029eb1ffffa53ef3eb52c94dff0d4f489f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/5afe653c526e35eacaf462b2335586538600e60e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/8354bbf557419d54c98453d7c8494f233002b12a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/8a0e6e8ba3af7eeb6a48ae0ce34f067acb60ac5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/b65e2297523b14e8a851f131da66e430c9ebbdaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/d39843f936271e89abee8f76d12e89e58902c682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/da754b10bd506e6a46735a32d4c64d84188fe712.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474660.32/warc/f500995c37e4d5ada50792fcf778621ca8a998cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/08aacdaa7a59060a3788a351d88232c226810006.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/09e4a0d4a8effe1a29c3466fef399915804c56f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/1df7d96bbee6fbfa0e584b0eac2abd5a5c42ffe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/2195653a44a5d6d86ba308044bd40aed254744df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/6b59748ba52943e69c6e3bfeca9117b4fe5aaa99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/7dbd1ce74c1fff0f1576e3b7626aab5bce470f6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/abcda5da64537b48024c879620ce2d475fe2207e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/d0fea3c92728399ae0477df533b2ba513b902d5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474661.10/warc/f3aad0753768ece3207ba6bae38c51e22fb12ea2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/2cd91b6da939768a2a847fa0a9b390ea40d39a70.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/3b2b0a4c0575da13b2c3950bd901f543270b5d8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/534eafa0bb084b89af5755a8da8b124e5727fcb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/572e9958e49e287f1ca9119405cff266cea482f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/5caa3557c2ad0167aeb7c6251acda5584e908285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/77e16c007667b8df09e8bf1d5a429d9a0fc7956e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/78f734bf190fa73f6b1918d1c7657b9bc64eb467.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/98355263bddcd6a43126283cb1c4383ac4a6f45a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474663.47/warc/b15a56a65a142e2efbf2f9266e69ece1e1717e22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/0367e33cf348b56ee32d4cfc0632a65a6dfbd51a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/9726bd0f5bf19982dd3d45dd9dab5f4cf41cfb31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/9cd4ccc38b440d5277c1dcf0ee436f90bead3510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/aca3463b5f9737161ff2035521f3408d370ce5cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/d5357f14fb76f5fa6dd0de1b27beaa90a4f75c7e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/daee4de61795b1a579bbadc87b983f0042534152.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/de0be6b42e4ae38afa9d1adba1c6b67fe1a544c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/ebd5f9b9546dec0ab8ea219e022e86ce0c5efac4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474669.36/warc/fc33d52cfe9daefeaa27d0d91cf58dc9e1e75b30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/01214549e26e1cee21fba8a071daebd51a12389d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/12dab9831f38262b30bf982adcd8fb092ca6f2d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/675e65f1e72b257cb604279f05c276795ed5c61b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/8d36f8e1b82fe5b7a3a5e2d2a8583246154e34cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/a8977a2ae5d85fe1766438864f507fc726bef107.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/ad51b1b4aaca56f42dd99245190618810f5deab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/bf53350a32a6b73dc3eeddd3f0df9786cea6e70e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/c757a1f67910007f7f99e694877889703c9b9552.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474670.19/warc/dc2ec098214fd8d9de6cb6e3bfbdbb9841ef7db8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/1f03538f85a78234e9f61181c8ec7a09c325f903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/2d0554dc960bfe4898885884ce47a92cea1aa7bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/3deff554418d9824eb27155dff659a9cf14606bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/61d7026606b0239c19fe0798cf8c9f2c672029d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/7859625ec054c671e497e7eb58b1ce7d51f05306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/8c6c496a9bc4644880dc09b495c48ce01a29842b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/ca5d5821321a25bc1a8de045ad81715768e05f19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/d71e52bafd3e595b1f51b99377f01450784fd601.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474671.63/warc/f25076f9d283f37b02ee7b63f40f83d4ed2ec265.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/09f67c24ac730a71a34eb16c6642b3b22a250fcb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/1e626539cf3c14925f3ce16fa68cbda92fd20df3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/3e9b2c6d15dde2be4a1bd3a930483368cb989a50.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/4487c28f491106288c79c198aacbd18bdc87bc71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/5be5b76165fbc50ec9c7a369be068f645f1cd0ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/74e5e318c521b5118448be27f77bb493364c071e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/aeb4de3a9e3044afecef5e7052832e184b7d9e33.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/d1f270a907e621db50bcaea74e7587baa41c722d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474674.35/warc/d3a48a8725d53422d4000c8ce7ab0f17ca36103f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/037310f44bcc8bb112ad1b82dd2c0718d7ceaf17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/575c16acfc14082e98bbd97374bc0d5b173d0bbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/a135a3e21018f4d6cc48f8f7528841c045b8fc8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/a29af557fe7759652682cde8b309ce702f28c85b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/c250a29dbcf714452641d01c6afa710f3f95a23f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/cee05f46275a90b2caf0f1f5b27ca7b833cb9cbd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/da83f73fa09c078b459052a2c6406ff2c823b323.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/e5a8e04fe5dc365b7db3703523c15ffb3745ed5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.26/warc/f78e5dd7c8ba00228469c357406ad3d1e7b9d6b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/2186cc81a8b3fd0ac659acfb273809cc7056e0e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/26ca3d51455ad89b6d6452f140f405481e1a2931.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/6aecd7b4a38d8a4377f065382bdf8deaab815332.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/6df35d53b8f7d8da8a394e05718716c526728d9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/98f30e33f1f9bcc58bb0d39b1fca2ec1d49922fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/a670639539998c56d48fdbc13b8422274a48cb12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/bbc66d9a46a420bdc79db1ad3fe647838d78df09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/d4ea7188f89a8a4d0a6da38450e0d4a663127cca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474676.79/warc/fe9c93fa666717eb4f7baefa3308b914dcb4a8c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/0eccf35b02bb9e6827b03b89dcabb2e2892e6666.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/638653101d54cd910a9878a6014be7d568b19949.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/89397bb3528c3bca8eb8bdd750d8373df60b9dff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/96c4772bf36133a57ac98d52446dd977d8d29e24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/a4ca6e84f423fdf80271ee8ed6441c4ca1ebe402.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/caddfab179230fbe8c050ef0cec79796d9915336.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/d30fbdda431a74c1b0cfa000a119a979d86123b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/d3dfc09a7f17fb302557864f57b1e28b747def6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474686.54/warc/f746631ca6f784705c70c4ba203e7cb42aadab6f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/4fe68a4fba949a33325b27a69ad013a454451aef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/60b8d6bacd17b0f893541c225a1a393fb4aa49e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/619622b9cd2d54c83640b73127d49cfd4905cc58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/743c4a9ec73003aeeb8226fd64f8144685df3603.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/a034126945bf31e21d2d3b581b2fbd4c7509cfba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/a24bda735c4f6f69b6d905382ce628f553877ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/b2162b147be259b7388e51a8fd4073f39180a23d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/db1104aa9bc93b8fe6f85f9ffa0ac958dd141ae9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474688.78/warc/fa4b7b80dec10a6c7bc8556f4acc5c518bfb1ab6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/2937025ee6cc9b88db7717738d391fb05fd74412.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/326dcdbab58f2b826c347eaa1ba990d7387d5d92.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/570796bbb46ab3797e3c99cbedd0ac3a654e5a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/c2dd1f7d71fa4f5bf36c5c69c53343c4ec68aa35.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/cc597371b3955ae5fc467d98448e0f7be262e955.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/d01939f5320abc03d6fef8aa6cc9766ab7c92427.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/d1ab1fc191b3237f0663cfd775f91b9d9190660a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/ec05d57cd28688a8d85ec070968ed1fb5e73c435.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474690.22/warc/ec24d271aa1d153a9544763c544c4c728fe88004.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/57c4bfc203e7d472d2450a7adc426cd6d73244cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/660e06d12e3ff674aa81ce53fcedca33505cc324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/7aa68e71a6851a8ea17b70668fa4e8a3e755e9ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/9b0951d1295ac2fbe7154583f5258d28b1d85a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/c0951e69c5b3248cc3375706542de47fce6b86e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/d9e9a8d1436914b24a0c32fa8e2b37529778ce38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/e05c95702a0cd772189c166bc50e1c1398dd75f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/f02926f53f2114d2caf73284f0382223de054e23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474697.2/warc/f817775d51ce16b51055e2239d8c4bfe4af095bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/1e17cab00aaf974011959c676536433b132511dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/2c7873b784a31fe5ea053570b682e614a6fc8a43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/610f1c2b8218719376fe2e5201bc29b7d64160db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/805c2700235f753e9dcd8da1774de00c67fdc808.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/95992114d9e90e9c6b0cd5894eac77929206c867.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/a24f486f3588acdbbe06dd70e497c2699512480c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/b8f7b458436dbaab3c9563e95bb36fc8cb9e9954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/beb2a941b0e46d20d372f59c4ba56670583d4781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474700.89/warc/ee81525b1f97685a7b36c4cb8aac9e0a240cbbd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/268bae2e34294ad01c2faea9f6256e60c745d164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/271f34e472ef7989d8423ef98b84db1b0aabcb4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/9d9bba6008c6d0ff431f2f91d4b78a01acb31c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/c49a5820718a790bf9c4d54cb1b5fe85b9ef5361.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/cd8d6b589fcfbbd0d26ca11f6d9224a670c0e138.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/d1f47659f700b14742f405f1bce2a431ceedadc4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/d71b5aa76f3053c71e08981b5de67040f506eb3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/e2f9abb1e0e64bb93512c5195351952ce8fdd5c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474715.58/warc/e3229defa8f479c12600445c2b87736c101c523f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/3839c51754f7e869ccadc6ac31bb98a3fd154c8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/40e2188fe6af25eed5081a9c9de3052a76c96961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/4a0abdf35373128253d0a4c8e55f988efa0696cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/4ff12e65a98d646b8bb1a2c0afe7f2f6d90d2bd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/b483e7a8ff82f9cdf3763b9360787efd417ff920.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/b6ab61ef442a94767b7634743d7ea188bb60dd36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/c7c29c124af5f88831d5910a937f100556a5d400.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/df49f5c568a559f2325c31aa3c214c74748d90f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474737.17/warc/f915bb55649ce52e50119c9182a929789d5cb7b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/158a0838d1309ad01fb30d74832bd80ccc413b53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/16e62aba2735ecde45c88353ac72f42ed7bb8185.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/1bce03dbdbfb5f56719ec5dbc4dd57e703aec6f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/3ba443b2d4f0bc18e18b8039128168f28631af9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/52dff5252975b3b19fb47fd74e4418d2dd9e6966.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/7a6d346e9d59ce90d3e2838ac9eb6ed3ce11e855.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/836f9e7b73b0f774d5b3ace98e31cc72a2f99faf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/94b630dc96c409ea9abf1f4c83e94060ed2ece0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474744.31/warc/cb8f281b3f66a30d4fdb3558e347ee35b106fae0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/2660ff862a89007b187cde5c39f60d700a9a8cc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/31ca94989eb4242fbdc8204e6469171ca7eb6c88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/626c52e5a640ac438ff64006977594ba83a911bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/79df8ffbfb8bdf449e991bc315468f78b8aabb7c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/7a0366b555a137433196e97e9d06c69c120a7160.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/7bc36bd23825fce6802361105d5c771b752a6da0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/947ce5742106c75150d58f7d3fd4b1e1e233d59f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/95a48d2d802824061017a5c13e8ed3d8b6068456.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474746.1/warc/b54b708ffc61c07421f0500aa7c1f4158ea14de7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/19d124e21e132f1e139103409f8e924e5ba5980c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/39221fcb4be7f64ecd6d133fe140563f7f9880c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/4dabf8e141efc18e705d0c4e7836c630f706f28d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/574f21fffaed9aeb7c0aa3a5dbee90cd1c05200d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/8ae3a7cadab7d797b8da159521c5073c270f4eb5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/a829028f77ac91244551dcca62288cd3eecfe216.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/af91e36eead44068b0dd26c57b7a783cb3071953.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/cb75931e01a4c0075e4687b5f46a28f6f3fc1c9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474775.80/warc/ec3e1f8c8ac6fc9970d6ba8d726c7c6cd9157d0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/0642e703de10c5a1cf1d9ab1ec6b7f1c5a443c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/13435847eefc9336b8704c6c425a9ef9e5f5cda1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/4c8be88a2bd377d2369f647984d33067024b21f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/7e98b56c2859e4bab6be3ab88a6550253d31c592.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/8d67320f167aca08fdeb97a0fd91683f31dd3681.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/9b2ebc5f8ce4a17b457f577d68e94bdaafb51637.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/a8a240994ad4240611a32c3ca68cc3125cf96bfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/c371b2ba57f9ebb26f9696cd7be7c76d00b8057c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474784.33/warc/e2f893be615b8bfdbbd4f171b79443a3a2c48693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/1f8fb03e0dcbb5641709cb5237059c571c319305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/4c8c921acc7819c828188a6deb0a935009c8bcdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/4cd57aff8dfdef94de7a482f049acb633248ff1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/5e7314117de54b9e177f50533fe300d88452625f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/6a3587b1ddb7d35c09d20395e0f4142ca9dd5f45.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/a67b34985d0b58c4cbefc25440a6c926b78070c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/ba1f17cd6924340a8b202e82b84f7f90ce0e2745.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/ee3e0e5481c535c1b7ed0f7126c3ca2905fa8eee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474795.48/warc/f6b791291e843795e2cf5402b57dd58487b09ef8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/053d612f99ae6d03f60856d4b6f945ae011471d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/145c8335a1e5e4dcdc0717cdbc8a50fc748b2485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/53b37285224d7935d1b85b097b19f4365d81ed3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/54f9706159e2ac989b608238669232225255e72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/5f3cf380a55a8a12523c2aea6de53a867ca5aff5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/68c84f01273315d3b2d677160537568855e754ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/76d683019f4e0028b64bd3bce22be7251c7392b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/7a6aaddbe02c48cde378c8374300adad204dec2f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474808.39/warc/c70ee9c069e1ff9cc0a77997d785e61c782dc6ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/129a9a5303f4bf083efbd1ed51a5bc964b912d6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/851b67c6b9af696fe09d88c55fae46e612e5fc58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/8709ef030a3d29f0cd73196826e61c63cb5792ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/8ddede3e7e7c43d59809b74acb6625497b8b49c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/9f91f906423375b3b80a549e00f86c0512d5ef5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/ae1ccfa55612734247dbf9623ea8439f5ff4ef94.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/f80139858d1baa17c359d2fb14f0edc5509fec64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/f89febbbe9fb8ed08fa1a694d82eee9930e7c978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474843.87/warc/f8bb8dc7f5b3b9ff8746546cdd247cab32a00682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/35487b7af66a154370d2a05ffc7a5b64a5e22312.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/42559d1cec69c9b4a1de458496a43e6dc676ad0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/5421d43562df4d5829591f1f3a9083a45f647093.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/5d69a6bbac26fd9f9f2b2125f0c3030e2384c3c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/9bb1f5e3335e40d3cb2f6fb78b0427b1b5a62b78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/a9a7a57f2dd41a2410f021966a4dd6123e11a0e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/cb09af2740aae176656e954eae8851b97d2ed5ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/d7b547e89f7e30c880488efc0db7af4d1c7d2005.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474852.83/warc/f7553233b155d6e17693cb7c9697eac7d4bea032.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/00f50181fd3a82fc608dcef55141d35b2a45bb8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/20f38fadaa9b861010b6e2b2d767fec3e1f204fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/3467e7b2d545cf3bdfd14e87fb33f0950ed764f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/35bbf77ce81c24a61e2dbd873a8f9a3d24c235c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/6ff765f984810d19fda68894f5fbedbdd71b0ee1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/794d36be174490e0fd176ba6a8f20af12f4b1203.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/ab785d1e39fc6f4c25d72d98c751aa5ec2f10a66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/aeed6ad8356d78aca97c043ee3c1d4c08b36c0e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474853.43/warc/bb1bd070f9548ca60af982407c3af0a1e83afa74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/11ce6b1bc7d32986a4ad8ff94921fcab76efbf1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/137016fc8e86a13b0fddafa0f0aaa32d2c797009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/18142ad66fb72e4e1698c041532fdbe141fe528f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/2bb43a2054cf9fd5e3e98acadd5224d4df9ff03f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/422817dd5cc7ffb0a8f0c827a516debbf932e2b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/4ea3d08dedce8e215a15742ebf7959e3123818be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/56f2685b71d1e552510461f74a9d25669230cb2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/675e7113995b247c8cd1920acf502ed17fcf5020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474893.90/warc/8c71bef8e2a8bf04c5a43d076cd7adeed5583b3b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/0213fed97499aa2963ad5675589faa174af9b4f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/0cd244b4112e6046547d08a2fdfe1a6e7df2de20.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/1a57d363d36738d7f1dcc2beaefa00a0d815153c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/684e2fcbec48ec7aa4a688a39fa0c0331f377e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/7a89545c5235706f606d3def2b5f6bc2004306f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/a18446f57df908bef5343577115925eaacbb5bb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/a9caa0c356978b39bfb55676dc817f999d2783ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/acb5ebeb29700a478a15e96fa03cea64bc6f0c05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947474948.91/warc/dd450b4ebc33ef75c1f46c193d287be7e65728be.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/0959e5bac845542284d9e33485c2b968080eb417.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/0ab67e61d340c849fd8de6a1cc626705578db9a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/23e5854286683fcdce359696416f5a9232808d9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/316ce38b7bf2e7ba686405dfaae2e834c860c84b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/468bc1f295b428a69e59c72ba6e47b3265dedd12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/5b282b06b1961f582952b075dd006924c8e8a298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/9c38ba4a1dabf49ac136ad000dade94e9540b786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/b5e4c2d4b47c8dc0d873a90cd9166d5da0abc471.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475203.41/warc/efdf2a8030c707dae101f3940617589c7c2f4a6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/078d0540d8abf7edcfa3869ee77bce6e73c35ed7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/34a659b258efd22bb117336941b5e6675d3cd46a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/50acc8518f413753e9941fd80a971b230cfa2442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/54b8218617a73f51665a30c70936d1ffa793db51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/7bc775d90debcc36d023dc95839f5dc90f4aaaab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/ab408a5aaeaf9120567898a5edb9375be3f3e070.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/ae82bc7a3853d287125a3ce32fbc3474227bbb0d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/caf3fbabbaa89d462820fd3cedca7caa8125b703.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475238.84/warc/cf82d376846d07a3859616791ec511c147633891.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/4bbb59cf63e025f6bb18a13712604b7c3990a83d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/60abc1370a3ee1c0c97f008ad5d4cf7d6af25845.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/66a5406206848f769162af8f73703d0a66356183.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/aacf74505710fc10dfd19479945b6694b4f029ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/bbb212ff1476c1bf707ce04248269c0c72f9e15b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/beab468bffcf96f3c98b379498ac8ecc3da6596c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/ce303331c0d855083563fac7ae55292509ce1baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/dc03be94a1024ee3ae0d87aa8ebc96525838edb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475311.93/warc/f018e98a628e0d6b89837a7f6ec42ff6e19b8fe1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/01ced0ff6db69753438100340f3b7add4ddb3278.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/1811d67d11ba9fb0e7ffe282dadcf10491eb9c1a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/2c7cba95298c9531e3bfabd0d72e5c17c71e538b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/59f739853785a525bf3d87f53a1aced679178dc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/5db0ca215a817842ac911424a22bd93fca130905.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/60fab71ea695185b5318faa5b9ee75709b0571d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/94f3ef974b42cae30b383d82ae7b4b23b0b0acd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/b890dc1d9dbcda1238b49a5c037905fb254dc321.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475422.71/warc/fb7deaa017d864a3628e0c6d1fdfe74c46260141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/23723e855ea3c38761ee775f08eca0fa8e4d8483.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/5ec57abac3827ea5920a122a698be0a55a1cf376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/805c33a3e2845191195e9dbbe4b65d04e0ded8c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/93249a451e8e4fa5fc368fb5156e0872de5e7648.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/9feba4ffdfb21905e352d4c6dad52626f4c66dc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/bc041449c5bf0daebd77259727a926c6279bdd62.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/c40015a5cc9f132951a5f49a01e603739c0a8725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/d0909d883da84309e403c9994fb417e349993eaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475701.61/warc/ed52fb7ffd3000647491ff2972b18b6d7ba80584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/0d4fce2306404383249184fd78217ccce664f0a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/1571ea5ce81d7c354cf6b4b35ac0243d4604ab82.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/4e6189c40890ec1656ceba8c0ab7b98126809951.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/7ac993f964ec5c0fa0f2aab74ee2f7da1fdf525e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/95e70b1d825e476a634e8a08c5b1a366d1fac9cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/e0f545b90260820397eb9fc06ee4a75f61a9dbd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/e4cb086fa050d3bf997730f696916a9256fa27a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/e6cc3c961c927e89e3e559367d2383708cd1f9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475711.57/warc/edd0a7f599cd148457744bce1f6c4031d51c6c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/2dfa58c1309b0ed0971a27ec8f9e6805c5ada1c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/3489bd254fdd7c3bc880cba6a92c46b2742e86c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/49c5485dad2addc068139168d62724939862578c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/91c1b35bda2b089b7969111f44ee53d207169a31.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/9f5e24607c674acbbf83801888eee5374e7be03a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/af0cb9bbcbcf96e8946e7869969e1fd35a0a198b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/c2fa4bdcf381acc5714ff4aa03f8cf21885b7929.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/d9298ccab355435bb7bc8c684767ad72a18e3cb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475727.3/warc/f836de175b31f8c963cf3fdfd76a1360a25bcfcd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/06e7be8a9067a599a4f3e867b9ff40f0e466a49b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/446224cd502fd83cf1744e9380c11246508eee47.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/63a56d267091f63d0f9c65d42cbe2c6d37c2c775.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/d8479f4a1e3c59692a637d610d3350f09b3fa2af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/ea7ee9052941ec4ed730c769cbf8c530b8c0a980.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/ebfce2bd2fc14cbd31ef6fe4a3ca30e00148af57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/f7c061ce3fc5a31ba355e10314919fc0fc68c8c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/f8400ae52d28ba23d4f04cdfe415a50a88450de8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475757.50/warc/f883099625e66b21eb79f6e2631420fd3bc01a02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/1a749b28089339800ff5ec56bb15c744716e673c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/1bce590d1acfdfe95643602dc0630559e7249c1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/379e1ee4e0361199549becc9c99975897c2c7287.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/40b7468b8369d891757008724b742dce24318d84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/454b83988c757c85a086172753ae8a276ce68b79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/6b2ea2a9293c7bcef88233ebdf496ef4a3f46192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/c6fef6a9f1a4d7420b71f9e3442bab515e1fa442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/ed9638f7bd8b73042d3554ccd505105f359492b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475806.52/warc/f268291b44d7f3b97aa15b8b07f038fb1420d423.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/1a2b0822a9d7e636dbb8cc6bc5956329a8f95998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/3edffca512bf693a40e3a8d6e78bea73789c1236.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/7cfb76119e63715da23b2732b7c85c92ec689eff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/966f07b8a8973b22fdc711af89c3e65245f80122.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/981e5f3ca5c3a84c44351709b05b0efd28cc0d2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/c596987ecbc56bde944832730dc58a52963ff22e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/d3743ff423404767ca3df713cabee6487753d3fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/d6ae035e0aff2b415878de2281d585b4e7cf905c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475825.14/warc/f292b395b568464495df6bf656121b7847e55dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/0697588fc50cb5ee4995cc3429a332f910bdaa30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/4b9b7b22c958f246777329b4bfb86f31f08a555d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/4de6afacefec9f6512f55c4544d7740cac3647db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/65dca1e2a15bef6c618a3659ed3c842d4904f249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/7e0c6539e0993cc75c8d759374ec02f836d2cd8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/a3083130bc7d2077b88bd0e1d23a63f06ff71c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/b3b0a32538f58776a9256f92edc14d3069da3319.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/d4a8338920f9001bbfd84eec4d88b2089a9c7491.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475833.51/warc/fa73faebb571492897834bb2a07dd85af3dee598.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/304f4b2ccf0b918308ab39a84a6a5c5f388a0c38.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/3e594861191a38039715799a84acc87add56222d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/531da3f48ff6d4600ea09cb3d485c097cd2377c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/81104d71c3159b5db0d7b208c1ae3e23450243c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/91af8ae9bbd2ebdfa7ed31280f06881a76c8f0e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/970609a3dcd7736a87872ef9da0cdae065e87fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/995ee1b9ab373e38f003fab296380550dc3e2eaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/be3b6ca3dbb3364ee2b7179cd2f0d9f5507cdd1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947475897.53/warc/c3f15de69cfceec278cddd7853bfd22e50134b2d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/10565e8b43b7386dc83e67516da1d1e2700e557e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/40930c0f483deb4e693252b11de3ec0ef460a848.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/559269017ace08de245fab28f2d4f7cdab36ff85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/7b3cea8384dac35b08bb9b17c2b17eb365e2104c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/84582da1c7f681e6bf4c969beb789ca8f9c583c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/9fc375d0bb32b537693b780fef890d5851780959.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/c1a9bcad96fd9745d4b8214a0298844c2b536d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/d2dc4bcba34bd836ea330460353de4ead0a350a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476137.72/warc/f8b1431ba1ebde56ad2757fecab9adad3103dba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/45384d0b2afdeeb4dffb3745d9b4743e63ad2e09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/5c5d4e118aba8557fc9e68b1be55f61311208671.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/85238ce11d3b53f82e1d29bf7cbb8a2c72bbd908.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/a4dc99813c57a4ba92497a020f25457f37b55c69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/aec9ef58dfdb9bccab0aa3d80e9b83bdf0b82248.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/c14cbecba3d8e3d1b4fe440ed1f68e594b8c47cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/d093efd1728fb7ee227a9ecddbf560681cdce6e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/f0792f9e7cfe776c58a9a3815851f49a5f983abf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476180.67/warc/f19712d05b9a8e67e2a20befa7a7814f3a668acf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/2c64b9d54947cba42639853a5dcc01826df9334b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/3a2c5d34e5cbd8e75924baf297a4093eb1a813b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/46349b7fd8869306c7c2afd89670025271230e63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/4e34feb7044ad6b598cf1975a783e1154b31d3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/5cf32de8ece19cb61281c665f989428c6437abdc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/6b09db704d2dc3bc804d58f017f8bcbc32c37034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/94732ff94f0513454b5b4013f1f53aa5c5983f36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/a42ead2f3382c5fa076d9638a5934bca106bb17f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476205.65/warc/e12c26e90306c7df4956c5858db57bd1687e610a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/046a43bd03e9f9e82e7bee9860826bb80ba7e419.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/29fa9fa4f2dfff7c764198a4161482c73e6d501f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/3ddc2ab3260a7b471420dc15825d6cb54a772e88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/578fe051786349f23e46f2e19df635953157b8da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/7667c751c8459f933aa60bc7e8f65d9c388afcac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/b96b66524a2aa7ac9570829fe938ecb41159355c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/bd7e787f28c56553f53ebcbbf1876e57f0cb8df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/cab01e19a22ac2b1b354105c4551e1b3edf0aedb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476211.69/warc/cb2135b10ce868814c1ee0e23fd356a70252c055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/16c383bfd8c4faf54259a31ad4ec5d5c5cdbe0f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/2618e9454f5fddd3fca9ac5d736b753bc24cdfc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/35d280c54e6daef1cb348f562137f223de2da1c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/7b724ea01671be8bceb39e2c0240daf779f5ce84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/8430fd3f70763f5c88e84763e9bdbf4881881c6b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/b19547b01f6a1dac5390cfbddb0fef174cca24b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/bb0aa72e883897a11d58faf866f28f4a5290a9d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/f4afd36613fc1c055336c1461c719d56dae4aea7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476374.40/warc/fbb97926977df725789507427c9f3629c6f9eae8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/2dd71bee6c44e3ba5b2c675c13e0dbdc33ca598e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/526e5f72d0fa0298d6c5eb09df176b275f6f31c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/5f8112760e0f0909f04ad5d76a439e2864f7aa53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/6336101da9976749b67e345b3e04ebb2ebccf5c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/6e2fc2e17e99e22ee50e2d059522af669f4562a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/82a7aa193b4bc1f8d0133cc2c30ad95234c0c34f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/c3f40ab4753a275e19512d9ae3a533ddce27da2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/e1f060e04ce195e8ecb072defd08294d70b0d5c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476396.49/warc/f9523dcc5497fc10df426f4d4f03a5408fa4149c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/32d995ed1467318670ae824391180520905bce81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/3ca9dee12115e938361841691783d96f6b14814c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/427034da886f020bca782de72eaba1ae71691537.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/55dd0a22f91d54abbe7da3236b37702290bdf4ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/7d903fa83c00492b72d4316b01e021ecc33b6c02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/88a1ae9f6eb5bd91568111a2e69088579e85fb2c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/c1d3e2677688ec3f10b48126a2d50bbfd3140b6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/ef4df195dff436414ab7392251205fba65d7901f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476397.24/warc/f6b8e7fd219d5cd161a01097b0894a9949b16432.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/143430f6d0b86612aa8d9fa7588ab27519e23b26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/1735bf940242edfe1601e27d1e1c68c1bef95a5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/1ae5459b971170bc1c4c19e0fc84ddb826a6c6cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/22af3dfa9f3e78dee42ab656df3d20239b5442b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/37de5dd937651fc44795d588e8a55cba625366a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/5ea68a2da4aaa0c891f0a77c83ac708f42540e85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/921069b162c1cc94ec4bb2fd29a9d016eb0435de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/afb2a8f845190382a13b6ca6f9dca867c8e6f7b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476399.55/warc/cf780be070c4c699d3af17d61a4e5c3c83f9a859.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/1ee38f29125ddbea9e8ea8e828a4e6b565ccd5a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/3e130c6f233d0b600097a4913a276f5a105b426b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/4d0f2385f5c61e72d8260578220a435cbd2ae99f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/4e648a11cc9ab7de05e5c62c130c62d1a5aeb141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/59af80b85bb819eee3eba131bbabceac682aa9a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/6cbf240da853d5cd892ee8abf9df0e929cb51b0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/8bbf5607ed408dd91ec984a6fa522323c8f83e04.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/9151afed55cf0601be03b1585fccb596c513de0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476409.38/warc/a93f73513bd27727f4f056c5432a1a097851d786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/1ba62e517ba3df9a24472de49cda93cf1d15564a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/260e2ccf261c431b9eccbee56ef8a9a05601b68a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/2e46bee964d6d4e472c1922a673a1fb878f5bb2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/7293d1f0fa0cad90879b038ab02f8849eaebb698.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/7df3757c72844621aad50caddf8f926e67a49725.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/82d69e6187a42cc8695120263cf3e6ca81ac8ad6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/844bf85a3e9223f2b3fd6ddc55ba4514bd6d6dd0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/97e10c40c24f9f432c34e5801df8c754ea5073c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476413.82/warc/d8bd29d5196808dbbe11e6b91ba4e58b1b6a1558.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/1794a027e1f00998c3a499e263d43c6e70b60a00.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/199454274f67f57f7df3474deeb7ce62ebd69020.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/28990b0cb0158795901694a40aa47ed2cbb245a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/587470d9bd09c910f5b669d8e21f42865259e35a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/6e862ceaeac46bb25b48c837415c7744da31fedf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/910eef746682b5880ce54b21a2ec544dfc926854.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/bc2ff5f0ae3609d8dab9d6d1d5d73e9a818daa05.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/c4594d4a978290cf7c38fa7064f082ea81074230.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476432.11/warc/fd51bc0ab2739d3fa4840885b5aa6585f65c5ae5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/63527d109b2889bc609f083802e314556f485555.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/6583e96edb7c161aef121d3e747c5c230959d355.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/835a7fd64fd427366e5d8e23e476f44e198ea108.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/8ed0891a0232c9b445872beb70a2a0b7c330c1c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/9ffe8f4d3a69b3f825ce8c42c886d0c73b849c08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/ca6221d4e6aa04b6bdc82cb6bad611d64fc4632e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/e04079720c40196e251de8454e0d69ea46afacf0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/e615f98a0440ab066902fd3a30779ac96e1c59b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476442.30/warc/e9166251994146ef17fde5aa3a431b3a5a0e8521.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/1739e77bfa301f2a122b52e9e85bc825de2e31a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/470607b3c8ad50ad77a4a56f1c4d24fccd36f4ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/53971561e7aa70fd1894ebb7d51eadfb1760bce4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/6c70983700a936ce53fb9962a3b7b773b849fbe9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/78c1eac73bda72282688a4138d50185915658512.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/ab01165c20dbf5acc415db1aa67dc0e1487cb9bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/ba2c6230f9a077230501600f74e2d6c0117e8fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/f187b5161a344ae973352fd75951f319a2202cf2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476452.25/warc/f3b0f59ea88a1761c823e5645cc93f045ec086d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/05c0434b987ce78d22ad2355d84184044172bcba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/19495d1aff19bcdc513fbc59b67a7f283999c212.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/42855487bf79cd5f0b4ef71f094365fa9f0dd5c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/61bc2b2d2da669713992b3ae296bc57170836fa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/66bf968dd3b34a6822f62548eb5c09f2bde98060.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/7a0e8c547b1835fa80da9f39b5be854115580a89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/d250d7cc011999f70144e4d51e94ed238a8749f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/dc60fd4cdb17f2f8a493b927a1c49c0ff9445699.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476464.74/warc/eff980f84709331429e8da6d8bc3dc6117bc4cdf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/5a88f02e134506a5d8f98d6e76b14105eb3b5395.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/86bee1cafed92f400d5ded9587b9a3fd00b8fc77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/8d2051c75cbf22fa5c60cf0ab2b9c43fbb2f61e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/8da614ed7a5796175961447a06fab7aa071595d8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/ace808f05253735b39e7266090d73e886b748113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/ae18ffd81e99924bd6f264dde369f98199dbc3b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/b3fcfff0def9653e20bb195fc00aa6e11bbfe8ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/c03fa1d98638fa3d9bd841f615ad01743dc42aec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476532.70/warc/e8113cb6d64bd636ab1bff6db01d76d48a28fbf8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/2708e37d79004401a69d66645497c3d1bf16e384.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/38d8d9443af0f83f82fa724ba654a43453f97f6e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/3bfcc343bf19c45df6453d1af408cc90c3dd1022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/a1368d4f8b41b4803f13ac3701fa13e9842deb8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/a171ec97c02eba00c5f14891819f330e0ee4f462.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/af552da9e5e4ef8dace40e3e646f33157dae17a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/c743b9399c443bb94095716132bd9a73ce1ded9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/eb49c976aadfa3010bedc269e9a53361d36e1b6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707947476592.66/warc/fc4a7e03c444a16e7e53733d5b7c287055260296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/010fd461f6d1d31b968b46378796b513c747fcde.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/06db2958ea33105e69a6adb84c948781a24e4e8b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/13e07fea56f52dc7fa9456dc2766d7395c6a9302.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/504c1325152a4cbaaca09f3173349994c0207091.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/8317a06e762290b5a6cea3065d0967fef27c9bb9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/a33d3eb11e7047162649fc806354fe8a3cc390e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/a523f4a6c73fa9e36810311b752d72ce041e7407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/b5c951b7c3ef87f477dbadda7c4d7f7327ac7a11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948217723.97/warc/e7b789a318324606c948a656d94e4601e81935b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/2633583484c5fd5cb1a6db32765fc2b3513021d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/4482b4765d90cff7487a9a64aa0a13af091b7ddc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/4699e2f8f9ee813d24c3eb35cedc724399811e75.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/792a2dd98aea5735a73ed433d7808d19de5c3e71.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/7d74bc06bdd03dcb17991b6b95f16098bc4dea5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/8c73cffc3dc3b92374804a90356de78957d8c1d0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/ab7098fb109329f2428937f1f381de706d22ef8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/afa8cfc7582b67c8f656677e6b9070b85b75e716.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948223038.94/warc/b00ab69b13ec6b0a1cee07f5a54f608416553e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/05c295acff7c48750f201f667fc165202626b75b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/128e0f7ac08dbe0d38c57c33d53e58fdfbe18250.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/17c4f6ef5131e6b43735f0e83e3fde502b7fe15c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/188daba42ec7a730dde816a6983c91351c9db5d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/1c5896651ae3d9b1f0bcd8e05d4a470a880a42d2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/3221d4ed5268633c5f14a3df7cc3934491c89a8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/396949856c92e9d20735fefa413aa60725d63972.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/c08af2a11685622063eda13243e00e1bf41f1c9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948234904.99/warc/fd69145a71c7572524a68c1c9a14c0c4cd30b504.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/1a5fc46db419516a1b146ed2b7815d8a6b98946a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/7e8455eca2694ae2724f258cfa20d0ff1771d98c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/a0dcdceef31380e03a19ab2fc5d0ef8d8e3b588b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/acc9bf18fe309f19ec601906a8d62ee7eb51856a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/b2d0f298b9fb317d2b2d207499ec72a71a267d49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/c65ac0be58c2ec7cae2874d36794fee28939f472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/ce7c87fd9c96dd84132aac3724c56e40070096a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/d0dceab18e7416a5c3be13a426d220612290ff5d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/1707948235171.95/warc/e51bcfd5a31fde11bf91c04e53823db206408121.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-10 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2024-10/mix/input" + output: "/tmp/cccc-CC-MAIN-2024-10/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer/CC-MAIN-2024-18.yaml b/configs/cccc/mixer/CC-MAIN-2024-18.yaml new file mode 100644 index 00000000..9c310bf4 --- /dev/null +++ b/configs/cccc/mixer/CC-MAIN-2024-18.yaml @@ -0,0 +1,1014 @@ + +streams: + - name: cccc-CC-MAIN-2024-18 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/0bf9638f4b669db03aec903886877cbb7855332a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/1b2735cedc42e27b00d246200a39426a9ff1031a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/3f546d6e5abfeaff58afaec4e4e1fcd34857d136.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/5298d73cb75586c0a61daa41ad28ef901cd474b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/8731f7d9a103a87c547dac087eb48cf12331afed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/90dafcbb6ca464c09236a4d46e6a5d17a9802249.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/9434b3569c3af3e3dc072c92c5b1e6d070365a98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/bf04fea003b56cb3d0d192a260bb95d22a268459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296815919.75/warc/efa549d9e990f80d3d09e6009436750b3ba3fcef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/35a68a6ff09ad5d3093d83632ba38dca0a8facb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/3da828c13d959a6f7b723c776ab3985cf3022df7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/61a2c91760892865d010479001ab122609bc6ee6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/81446e0eb751cf4e41983272411791f218aa6705.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/a033fbe0d99ef58f8840dd31d885a780633d28de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/ea841001e27b0bb07ba1633007a72d0692364fd2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/ec47c83ca5b6c388f796a742330dee9329173b1f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/fdad967c243448c02c63b683be1d8ba24e914864.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816024.45/warc/ff78175283838b301dc5525c6493ff121c581873.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/01add4422b4e4ef13626e8bcebdce3341ee17c80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/192a08f8d2f8aabad80fa462548bd85a3b749843.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/2d1ece5ffdaedb716debec3ab1833b37538b6f5c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/4ccdf5008c2105c76d9c7bf4df9a5830f2845971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/79a5892000302fe95b671afa1a6994cd7c7c3a55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/8327719fca766aeb22cb092d409d4022d8e23fb8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/97105fae66b945dbf6562600e678775df46adb5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/b2a358b4f21f8f9c4752fc6805223c92e9c6c525.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816045.47/warc/e67263eb7efac9c1c66fc10f6754b90bfd57523b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/010527c8640f2fc363f6a6dce4b15a1c496983ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/5c4bc746c740921a21f749b973d1f8a9f9a00933.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/6218ac543a135fdc6bba2569fe0a18169a20fcbb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/9414c80cfce63d87f88e887ac734904cd6ce63af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/968e2631b2df64fb829db943bff575932859cbf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/aad676b86e211aab6c087052f0e2df8ee5a9107a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/d48621ee317a83c9c98a54edaa4d4e3c4bd563a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/ddf414269f5e8c1e18beba4ed8468851617f87b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816070.70/warc/f3c58649ba92782f0144dd4f00f92a88d5661c6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/00a1d6831a9ad2b94efbc4a9213c97baf42552ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/21dd7da0ffa531d77ea97dcdae106f49f6e02ae7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/4237df16c43ad8ad42461dad7722d7e09d1ae8f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/9768ae115b0d04f117e38b956f623bbc6637fe88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/9be98401afc60e2cc5c166c9694fc6c9b0868f39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/c365ab5fdedf2df2fc46e0e8f9627a4b67e2a461.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/dcece95d7e39cb5c730edf45679747961dd89429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/e86995cb879aa7c778a36ec1caf66dcec4663304.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816465.91/warc/eef725d0a1f899f0ce253da26748996c068c12a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/25f46f19a01c0085c781bc3ebb5b74cf7025135e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/45d2fd57b264c6800b10d6fe45d3afe4351d7730.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/5bcad67bd4400238e19379ca08b34c039daa1815.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/5c1adc2b35945dbbd938433fa1f6763f8d74c18f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/6cf4deac97266b3b504efa5b227a7a99bf067d3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/875cc88cb889830ebe2a43e97816d6a6faf857a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/bb35896e59892c8939bd94dd944eb50dc95d02dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/bc9da75392c52da33a26b99b303ee20472335b15.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816535.76/warc/c9a664f07cfd886d8e5033df29d7b05da1371df3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/1c73070b6ca4ff7680998747077724711ab7a7eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/4c75c57fb3644eb489d31b306e06be7dae0aa164.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/519e6acc9da83415994169a0454795e28699c78e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/710f878e683b982528a33037472faf57f824788f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/a442b81ed1cec232852873df648007782cf749ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/a668574590862270a66f25e53022e063f1461b27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/dcd26ecf9baee63d40cefea6c33d6f623163f5e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/e4f010770d26a05807923b9cf27fad53152f73c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816586.79/warc/f43bc4b9dfb38bf8d766fce8564950447d39b87e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/1d82a6a701104eeb3043a822cfd80271e4895c14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/217c5c75a550979ca17392dde9213389bb8ad1b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/24b640c55079d0373630837a7e61cf727a304099.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/25bf4ea19f39b016ece2bdbe213c24115f027a91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/4bfa10be979b1db6979e3f412a81f5f6af78dd3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/5c0dc489bcdc8a0b52d0f156c7b9e261d9cfb6a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/a25d677c03ff8df414f6759fc95017ffc4575b5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/bf2c40526a57d3b5f0ed32b83392ce02649ef454.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816587.89/warc/e269255e85301cd8e86620feaa6f5fd98d74285a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/0271b18eca21e94b3766973cf2c7b8cc5190c3ac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/0bd8cde29e490eba667d72450bd5737fe99e9b0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/7277e16bc709434c80f6fbb000f77fc000c4c194.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/749d3fc4f24a5f93ecedb869de697a36566354eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/79aefee59d97a7e8d67d76ae626377345a2ec231.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/7e0165d18ad130024ab98f195e3f922d612a4f72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/90fb97d3455a0820162be1df6517841089b40383.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/a14c25a8be490173531822e4998b3c6feed68c2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816734.69/warc/a8245dbd1537b94fee41e60c7fcb1a65312e37b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/2986788f4bd0d4195090343b4ece6bbbb2a67e99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/3886869e5dc888e79710f7e4480cfe091257a41c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/3c6bf9984b5425341dcfdf8571b1230a0c04f97f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/5ab9b523a3e6c1b648e9d2fa5b62ba71de4bcda6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/6a3df68c0f0741787a97e9650c1d1a4fc8fb5342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/8b54208edafcb87386c8a25f9e796f2d65a6f51c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/b0d26f16e3b8f6f55cb3b2046f09bb40c39e0fd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/c072b5f9c54312557de65215f1efb05247a696d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816820.63/warc/ddb6d63bdb1813e9c19dd5fae168d5d582b47c74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/19340d4805b2368ce81db03242f10015c8c827f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/20a5b7e8e78ffe379a6c1c3b9092066d0fd33226.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/3025987dab50839fd149fd667ac12e22fc1a25de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/48706f40dc2e6f22f89de5f76ec6ca0b4edc8e14.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/517327b4c222dd7a38179a84ffe1371969c4f754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/7dfd8a008ba91d0aa0d299231fed6924a3aa98cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/888c5babbcd4d0051af01053a40f0156954ab7af.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/f0675ab8342501e0daba2aeb561f62fc91c1af1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816832.57/warc/fc9ad03f4fe593a66d7ddffba0c2cebe55e7a628.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/0162bce352471b09fa5a001d29a56098c99d17dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/29ba9f097b813a7c8e673227167aea51663f69b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/44e92542f049597dc9ce528654aecf6dbe917f89.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/74100051fdbed0d28408cf7debfc76d1b52ad34d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/b65e8d173504bcae54bef5a9888ed43a8416b7bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/baae15d55574e6e865451bce4792a1d18dc4fefc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/caeafeecfc07be87ff0aa53e26e12c7600681cb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/cc6f6df0af38165c441b5685e72ae5f76b9c1374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816853.44/warc/f6b6e813a1e885779d2407dd2ecd050c238f36b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/14df53feca5d666dce16771d8764070de01f918b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/4b927a48f74e994b85e859e8381df28d8862419b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/4dea61f70c1d7b2c0cc3f494a514096abb39cee9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/60ceb0ec5feff15992bf89ad1383c8118add11de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/631b24823300ee526d8304f4feb7fd817237b516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/6f8283c5dacfff42f8047da72c1aaca6d9243e3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/bcf36bc77ebabdfac813b087d0ead382bff81b2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/bffa93fe6e3b72a370c0b47616c210db12f7c588.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816863.40/warc/e733614c1e1ff477d9347d09047693eccf08400c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/0a5d3ab8825d11f509c4d18f6638ca19dd1c674e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/24a30a62508caa699f0c13750d17f95d01252840.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/43511692d4b9826d6cb2363411d7c49faca88d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/9ba92e614d4f3bc80299457b8f142b63780ee526.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/a2f60d6cfee80d31c7f9e4a0cfa949af185fd1ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/ce4e0b385980f812dd6d48665926c61f6c1df4ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/d6342d4c3e637cb97f8252fdce22e144eaf9f903.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/d9a8a5c62452419607d749a620f4dc3f821686ce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816864.66/warc/e6ab411f861f2adc33db0e7a6ebef5936cfc00cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/0a984796a0ce103e28cb198981af4a4b55e14305.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/0ad44aef8a4341dccf1ea8c071eecf69cf990042.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/38b05f7e353d291c637d8ec6085148d7643f338a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/4a6952f99833ca52507c86b4a1a4fdc4cb11afd8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/688b2c1bc96da40c7f0738b8020d4c76a9f2ee7a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/8e44301758f17f5e42f8b210b8f3664d57763c54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/b620706a0f39eb5d1491ff4b857a8d9434d76e5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/ca6244cd655c9f202388b3047b229d08640edd61.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816875.61/warc/e47fd6dc906172b090c8c5289b85d0c27b19ef69.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/151f2563243576397afea6940a8b0af1f404bb27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/2876a4d85463401465397c88a93dbfd4c40f0645.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/3d52114dc53407ec9a8d10393b406308f74a1348.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/3ff09a34fd1379e4944b64948b0086fbd67e66ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/784cfd325f8d524a966b6daf301752e4da5d4fed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/82031fb831099a776d21bcbffbbcc032724eab2e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/9ecc52496c43307380830f05e6362efb315c23f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/a28c01aece993e4228af4bfaf4d710d5a06c0e37.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.25/warc/d8d45b929a26400a1ee0cd80eb6c247af90a1ab1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/01ed9944c88d70fccf0e8a106c4fdd1641a73d1d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/27059d505febc2a53d198a1439afd6d3c9536a51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/27e8df37ede8aec15bf65c9ecc887e79ae291f23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/299f59374a88a390b66e830f82b4baf818fb594e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/6c705805240538d5d5277b1950a6a2528acd8b83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/7768fe17e5d66dfffc89ae6278c2612f018a19c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/c2a303e92175da5d33ef85c9f6f66886ba6e2292.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/d1b932108f62b734a99c1a3c2e0ebd623f6380da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816879.72/warc/e48006ca5e62be8d121c9d52f6add9646e571a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/097f0c7098d85ca0ac7777df04d71d6524b552d5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/119a67e15b653b465f78cca537579cda690eed4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/3c05cb05641bf18d9e7478e47b81db7a1a651b3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/9f39c49999323a582e46c65096de314500e2fe93.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/acb42a40ba4dc9660e803db70b347b584c00126a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/c256a76affbf25535732f636d2611ffc4bd46429.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/da60d2fd4af42242463c0dab6e96b7d5e6801192.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/ef24a11f077eeb89e585f9bb349efba1723b2000.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.19/warc/fb7fc1d674c56e9685be838281230a9b854e5cfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/054ea892b343b29ce6b9f0cb3e88d0dd283dca90.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/0c3e65619b2f930ab2989a5ddd79f73940c0c803.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/25c287eaf9b6f62ec7424be16242fbd9a66c57fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/3a3149247cf1abdca1ea498e08ed38cfa257358a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/a1a59f5b07c56adc6640600b3e7ab8a5e942e8cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/a6244fe07d61132776a14d2c9d24abb5248627a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/e8633e4a3e2bba671f716e89bdbde40124c2f95a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/eeba257a0eb9294a58616f3b7c85fb756a8278c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816893.9/warc/fa184ee4314a91fd882cd8cd96c81f0154fdeb30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/07ac310257cb684396aa462eb079f6344c0ad13e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/3d6c12fc463f958142c3b104ed074060035a59c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/446ffaf8effb7507b8d63bca1536c65830d03584.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/4c03709ceee24b646e6ddc0f56b38c5c2118fa16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/7b39aac403ccad6eece94929c05240f94aee3d19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/b8cb81bfcbf8674321f047042fef024b1246fd4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/cc828861f3328aa93650e12abbe96a1908ed4330.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/d4859db5e165b87fdbf6c54172ec035fb07d16c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816904.18/warc/dcbd10e9598e7c2d57fc770009e0cd3c350705cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/1ce36fbe9a33481110e65c8f23772b424b724875.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/2996a27161e42a8c351c786857a8ab49de10e232.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/371660591173298f4c25ef839db03f7173c5216e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/4e078ab7b6dffcec1702c12516af9d7ee4d01a2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/6b1b06e28a19fcf974cc27828d80be15fbf2b4a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/a52aef65b5771eb0be44148e51d86a4698a10d5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/af30f74c3a5967e7c09f822352362a9385803c36.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/f370f78e08065cb47b6e73d0688395c35e419af0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816939.51/warc/ff61059dcc686b1bdcfa7f6339f90a09f4bc9420.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/32f0259f568c40d15f134f918d9a4f888c04dc9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/8806303a85f4d94d315325f6003f9afc41efbb2b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/88ac1f0c571dc2046c51e920f171e8304b981758.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/90d5a27af6980b0bcdb0b240ebd832d17d2fd693.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/97dde0bc1910933f98809e3cc7e53ef4836b5cb1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/a857afc568bae8310a2525f0f582cb5db5040e39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/a9490a7c05754435c1c7ffc2b1b8928a7cec6059.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/aa130df2b679693c9933c1db0f37c32cb442d7cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816942.33/warc/cd4adcccf5583c01237240554ead86701fca9fa1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/22c1f321c14886ad3379749efa4b803a32d1e285.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/2bb8e2a1f3eefb6868e7f16a039c1024bce3dd27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/39b21ebcda9b90439b0aed3528608264c036ae72.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/4b21de647cb8020f794ec9d29c38d3cad536b9a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/90f97abd4d7a16d6778519c4c492123c0ee61fc0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/94d609896d313cd1cdc70a79cd34e9129a0ec7de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/d880283649d0e5d0360e582445bcfb0c73ea5895.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/dc124fc281f604ab47a438bf4ea4d812d4866d96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816954.20/warc/f22cedc208c57195f093d1f33afe9e1e46274fc1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/0b3ec5d865692449a959b6defc1344daadf86f0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/2d091d49a0e7dc1d5220504ecc0045462742f36d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/2ef7aebad6097d2f565da7e06d0e690aacd42feb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/34c52f296311af80be60405a8177c9ff52715376.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/842c05e0973473fc6f694d6d2de3e2ac1ff486c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/8fe47f069659518008f7c76d9379e23040924662.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/bd21f80272ab38a3b0cb6b89a7d543177e5b9544.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/ccf84d16f1610b81d9d3872deac587abef97ddeb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296816977.38/warc/e9b6e285f6f8dfd5c8885ffb74d28e3feb476b4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/12d70128ad4a3ab71d2ad27808c756a63dfa3e57.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/38f77f699d7364201476e0f8d22c64307c75b85d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/5ad6252a89d795cc105f1829c119e0eec5dca9a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/bae8b3f199d3b90a43e99358a748bc95f2ebc6e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/dc5ec6e1772d20cfbf40a805c6138976b3b33f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/eb0a14faebd0733b6d032be5f9e617ffec52d35d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/ed03581f97752e1ea974e474ecee609f665759a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/f0113127cd3b30c951f719d0f8ac43a2ac4ba3c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817002.2/warc/f3b307d8752d7503311f41ae610a797abf2c5062.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/05e745220c50ac52f578e958fb4b703a63cdbae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/35fbe8efd8cacdf596a104573f02636ce48d8c9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/5411472ceadae080aab6b1715f2f3bfa87cef857.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/6f3b125e0df312e77b06b620c7f0d15fb3e81002.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/7abaeaaf6da635335036af208c45e7ab7ff6e3a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/85a53a1413e54d5af5d9a08043cb7d09e980917e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/c0270aa18e6306d4235a707b0d414372bba18ffe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/e89ef899653ffa205a48670735b28929064f2aa0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817014.15/warc/f1586dfba29318c09e12ab90f31ebf0541c0fe7f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/0475a61f956bea8d546dd3bc20674650d1128db7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/36ec1576787e9fa53933ac7d1c72b5aadef01a24.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/390bb6bef904604b97b9d02f1d1ab96817fb644e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/478c5be286d9b6ed7e1e1e1585e9210f156ecfba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/4d9c53e64fd87fc65d5c9691aa7ccba1033db18f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/5ad2a79ec48d29e95ca34bf23586e81c44b5bd4b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/a5dace1c515e91a002d6d8d7f71ad6213a651fc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/a853122f0353a65ccaddb3f562131e2b9ad76f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817033.56/warc/da8a2af3224e8f5c3fdf7053c004856b4d8288bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/1012a7d1dfb270be25fdbb86eb5c01ef1c8f4e28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/1f80980580723ede911f0adcf8ad327cd38c7b1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/2e9616c776d1a0a22eee978f98ffa9f02f45ebb3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/4b7e2ce83c71ebe4064b8b893d3da0242ea153da.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/4cac1909727cd7c571dcd42af50cba782f5bfc9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/5328cba2f24bde54d417bfb050d598c1dee3a961.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/71b891213cce4befcc18aa547f200bcfee13f050.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/8d4a822d71985feb28e356a0864c91a8b7c22ca1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817036.4/warc/e1d86103016e37d0b9c8b13f2d2c73964584486b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/0093d29381f7a1f1af6a8cc354eba79fd6834128.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/3628f4fa0432ded8adf62408f1f687b572f9e8a2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/406780dc38c88976bd4d206523e2ea151ae4d29d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/59b09d93491e91160790837391539b3ad7a0738c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/7e93c85edba1d7b89926ecc50abd6ab94455fe06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/8c919fdf39cdbac827f8ff5fd8071a20c8026e3c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/b276ef3774f0309c1969bb4643ecf6ddfa319bb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/f732ba02fdb4979d54a2f1f618ae3361a609a746.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817043.36/warc/f8c632c9b89a7bd202fd878d501d8656bba57a9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/10affa9db61417c6c651776129da815ce2a36dcc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/2971f9738abfd7265cf3a3547a8304818bc2fe9f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/51fc935ebbe9530d8852e1bc42df1156ec80f8bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/616c532f2d89e9f8ca682e3c08d5b12812e50f51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/828b7b01da792c7a3cb39b5ecad17efc24a93b6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/837f328f8fb6e69dfe4a3091baad04e2bf2f7ab8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/ead8df627d8002aa8c91f30b33813598df1db1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/ef10629e58248dc6b06431f59dfbdbf09fcf7567.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817073.16/warc/f1325b4afc35306b2941371037fd3bf1062b5f1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/1e216a898a56a5e21663f0eacdd4b1b00f110316.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/2a6cb39a4dfa97cfc1aaa9696672b320a771c45a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/3195360e12cac91011f278922946f7d07df0d3f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/5d128cb53cfd8d717e83aa4bd9441c09ba66a753.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/66b82f2ddd450daecb29a718c290a7b083ed3d1b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/716e891c5a11328b500bace3b1278966c6f2f329.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/8c22dc34d6c9ac6a0446763065e881b82f6ee754.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/de19c1174f285ee1b5bd94c285ea8f531f1bd5d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817081.52/warc/fada2592fd0eb4dffa95eb5f64728e289fbbd998.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/039742fcdc381f343f31e56991833c60a7148ec7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/1e93758cdd37a1f477315b4b13b1f1f0e62b9154.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/7ba20a81ab559a860635149b53e6585a5d7c17ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/ab3c86864c650e0853b73c3852213af564061b53.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/af3c624abeea0311f3aa09ba08ed34a932c10f54.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/d288a6605cf0f0ee517afe62a36785cbba13606c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/d2a7538e11c7a7bc05479059a16c9839a8a9b442.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/d68134d0519bd9d8a47629fc385d8c28ddbd4d1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817095.3/warc/fe00d219c1d07bfac6801888196127003c8dd1f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/1a8933b60de2b7516bc9d961874607969268960c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/258d161e4f889f47f1e85f30ceedcaa6612e747f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/3e5c7b20a7bf80cc36f2c68da03eaba2cabd640b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/4862d8d44e3a2061c7d23fad3f01e2f3d2b03978.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/50dcaa4f08f9c398bbee73d69d71d6f2fb53e7ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/58cf536847beca160a58a1b643e0f46fb03d6890.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/9ec7ce2deabab7dda4b725641d4413e4f9f378bf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/a65f28611398e244f23a28d6cace69b4fe12c60d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817103.42/warc/f531001780668085e826f10f887f042b121f1408.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/07e2705a2c13fed785531e55e01c03375ebcd3ff.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/1c787f6ed4c03814c40273ff2ab286d6972c5d5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/22679d690fe86bb555e32db64b9dbb8384d3ded3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/3c412efba24fa673caa0e6df36be59751edcf7ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/52bd379a00070824c4a6c9840695320f76be980e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/62a4bc99ffc587fad8a5d3f6b0368c2d2098f1b8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/91bda3d18cc081eac2bf028f652585cf49802f8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/9a4f9c32ae84d72a5ad4ac1122c0ca5a5b20477a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817106.73/warc/de7e67960c1ceba6a68e1c6a7cf04b198b2ee1f5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/2e1477e29bea71ab040f3cd4c89f67736e35bb26.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/38ee811a959eb5b1c7c2f51df2ee5bb5ac04eaae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/4d29468f662cd6aaf79161c4be10cb57a04de3bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/6885ed50350f90c9ffa751e92f9ebbfde2e88c87.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/77000e22f790c5408d806ac705b9b05afc64c40d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/90d1de21ffc52ddc8685141cd7f0c88c46b36edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/a533305e3f084f35cd06a2153564eb7b681d4dcf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/d64ec1ca1fb2305c044685ab030d4be8a53c0ea9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817112.71/warc/daede23be8eccfeab000dfcceedf6a50c88009a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/0938981c6cb37a0097e8dd9e456ff77711462307.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/2106ac25418a2077d2ec3f6d94c6b1c984dffe74.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/2b79efd04d39697d2c2101c8acc3c4d610260b22.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/9456ce57c3356fe2d9506b14eda166d8d448ba56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/b1cef4f8a454532c92ea57a5a3d071e66aaea301.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/c7fee57e483b03ab3276c3508bb169ba4f3e18a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/c9188de20bcf7e014e2db3632df1a535b6ade5b4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/e45cc81c69d81f8d08a998affcb13cb322dd031e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817128.7/warc/f607337d3aaedd8df3a1844ed837a4e79a22cf58.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/2d6d993568af428eafeb59157224cebe44a7cae3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/35db600cc09ebbc19da9a751e308a13edf348f19.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/3810fb6a0cb6ced64e9b00e63fe4cd71a3d96b51.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/9f6b6f58aafe2095b944e7ad5b420d47a06ea28c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/aa3dec27a9bc2c4d0f8300239f968f96f45e2ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/aa84e6da5c1c9ff786a4aa16d1d0b97c380bf4dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/cb2553c61b593182bd3a0160e91e664071bd45cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/cc96ea1af83327a7c54aee08fbca28e56c4985c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817144.49/warc/d241574edbae7b989d2eb3fea33e67f0f158878d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/35f878553dae68365dac5b87f95866e03dfbfeae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/43b5efb88dd1eb2c437bb08393e45f89007793cc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/49079248225d77f28050e80117a499c374b875a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/4caf897bdd7fca3e7370abd5383540e287a07a97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/61fe8eaf0b49ecf01a6cf7a7b39618eea2aa51ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/65d23ef5a2881701fb79ea8755df2e733c4480aa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/ac101282485a8f9c1b42538e5407f25997a3ff06.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/daa119cb64d1c0b4ed92af47d2ad6ace040ad9c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817146.37/warc/e3bee34248c1b79810e62b932e3ac8dc897b20c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/1661f9a02ee1f0df9f5688a250c5c2f50dd3c0d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/1b38c66aa7e31fe00ea9b315444c9fdbbae35bfa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/508210e64c0deec9142f4ab92363c43bbd859bc5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/5e35b5465c828e8957ba2b556cdb2d96c73fe1f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/5ee31f14dad5b20502e6beabc21b2f27bc0995fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/64c272680133f113b4b1af0bb3a97cda8472eda1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/9bec187566730f4a95e50a10642b195bcd4e6abb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/b1e124d8e759913416582187c5849f7b63b22c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817153.39/warc/c5518a344f064a98c91d913feab7c9e153995024.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/1609cafcc54e43082d89c7ce3b3690051d5d7565.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/33a8703901674158253f9c8c18637fa1e1b21c30.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/7238f521bbb7bfcbe14389c5e1b4c59dcac828a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/7efdaa8c9a1e2c9fa126b99eda5bd78626fba264.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/8d8b10b9c565b54c36f4cf6f4d201f4c145739e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/968092629794a2e7248330b3cfb80303bef083eb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/989bc84b356f50cb0f08376f5cd80302bafe5a8e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/a0421cf8e3edca132a8253290aee915b32fe44f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817158.8/warc/b882897f8a657baedb6dc99e00eef8aaa8f1b2ee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/01bd60e0997bd5c4d31d68653c462259bec826a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/2f4e7566261f0d65db35e091f6d927a41966b7b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/37a9f07b18e7c56b533e20b9f82a9ecd58a5e7e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/48e70ae346fe02a5d6e10ec974e6bc7bf74e1447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/4a0d1d50d70dbcaa3076625532b6705751d7d137.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/6782438ad0612c25ebe2d7a4cca632d39cbcd21d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/726b89a7b73ab5e49fa9adc217fcd3889786e70a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/9d4cfd6f3ec85c1d5cb7bfdc255974caac60e6c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817171.53/warc/d32dd3c60c34f6252d7e8297ff9af0f6a10f58ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/000afaebc0218dce9935caa829de606325c8435b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/024d0ac3128b07bbeac9b33ef86e58c207783158.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/0b98543f18903c8e2c241536ec2ae648b81438c0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/272448b69d9f0bd3ad872ee817f60897000840ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/2b9f9aca926bd77ea5a90b296682c1e5fa871a46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/55d572fe3c4359d30c21446db45b46c81a7209e7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/c0be2d934822a8f4cc07f12e3972ad3344b67196.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/c8254c60c22cf50fa55b3427f5fc814350fa7407.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817181.55/warc/e41223f1ea3567bb239f0c9b65d71646585b3c5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/08f946301bd49a8ef130bbaea153f55e99ef45b1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/09f0adaf73095b2382e0d9c57cf1635c54edc8fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/1642f7980aaf329a8b0e6c15254f570ec4c7e8f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/365ca5d909d823c9eb7667b38266b083b8ed6a49.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/3a3d9ddbc94dd0dbc43364926c420931b57587d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/40fdd2bd99813c7869511b7d374b78d1fdc3db99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/b357082dfdd75da7e555341f110f9e1bb0c9c0c8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/b68b7dae58a7a3cf31a8dfe6b39cdd44fa56fd63.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817184.35/warc/e40fe277e5507458226eeea9440eea044253ac84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/023295cb467402439eabbed293a26da94127515b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/1df40141608795fe7be0ab0f72a65a7b2034c6d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/22dde568bf2557e389e7fb37363a4b04a7fe5596.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/3116daa5f03b69855713ed30a82ee982d4620f86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/7aca8270e37f7804492327b603ca05071a471a8c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/a68e00fc3ad457f9309744dfafac799de67ff357.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/c6eb78cd50c6650226f92f7102f690fbd09a7a4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/e60f0b10044cb08155fb7ffbac40ed1ee5c94f99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817187.10/warc/ecc1ad1963e849f6667a8f3bb7641aee6d418842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/1d74d0871da6cb07bc8b1249bde1615d678a3f12.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/50e224a7d3803f73a9f4d6c8b52d307f6f25fa8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/53dcaa5fa2a8064add64c239234adc197021c71b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/74f22e983cd1e24b3586506c97a5e9b2b9f08296.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/9668bc3cb0d126ca0b2bc9bbb7342e2154200dbc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/a37854199cbfd84bd6e575fdfe63dcdf21344948.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/acbd6061a40153fa9ec5adca9a51c682f60283ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/c54b0ea99f543a2258bc7dfbccc317bfeab82c1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817200.22/warc/d177ecf25ad50df735c880cfd58c159e02d74817.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/1551417ec150f9d149f9aebd9f24c26ac87492a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/3542b7c1ebb1e2c3d3c923acd9686dd264ac6ae1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/3e7a9089bb91f0a766c7be283ae11b7264702068.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/4ebab9c262f4f080813f21988b536229a5024f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/5b25df2cf5883cc326ea11808006bd0b136c9ec9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/60d6d8fd59d866795484cc4071370dc3deb2d7cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/cfc545f9910687a94cca6d5a48cc4eb913f0b8dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/e184b86f08ab664ed8f7da21e28305168f80112e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.28/warc/ea7b5bdeb17630abb6212859a5fdbad887393034.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/0120f3a928fab3db38b13e3924cc795cc5fe973d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/4a670e454cc677cc30518cea0145814b9eae0b41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/4ca7dcee7707ff285a0a25ae412b4d0b9a410b9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/63bdc7615fa793b168f1fd00ce771a4e37db322a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/afaea7d18ac9b1ee3985510259d095e508b5cac3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/cbfded285d549b7a3b46dcda6276d18e65c426fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/dbbbd32e992fbaeb986de7567cc450cdff994e0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/dd4263c6d4602b3733898d6fd640a300dafcea39.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817206.54/warc/e740f17a097b7f875f22a687ce600b0494fb5464.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/0f27508c328a45f5bb8f171a12e418f1552925f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/197da6c35984b9c457947ba358326efc21213e8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/31419a9a8f049debf00c5ff31de1a1498a286986.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/5e51471049f832e795c72b3a0fcfe37525b256d1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/80ffbcd0581fb68c826a97494760971db9ea09b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/c61864414e8ffd5fedcb7aa9bdebbc95e620daf1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/cac1bd8a3b5390845293ee2f0beacaf5a7412ed1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/ec95a74b40c26b15196a46ebe97496da71d21f10.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817222.1/warc/f6a51fbd9aabb93699fef4bb2a33e0911d1a9546.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/02c779e17a0c8b5acf566e7b1b9c307c65b6f88a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/35dfb23d30323853b79d502cba30b9d3d9ccb539.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/36682b4ba03e9d876c4f69b9a2a9815a6e111882.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/575af4f52f8e08de45d8f8974edccac1db934ad9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/8fe601232655d104de4b281d35fbe4a03b4cc6ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/bacbc8849815c845eb62fafb43f4b539db3eaf1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/daf627771147d6a6d933101986f928ed2dc5090f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/e7dca894df52b9f75253ed7b507da8ab3572b72c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817239.30/warc/f13c3f750fc39deb1f3714db5130f6de67d19b4a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/1842957f9015f7294b5ade83d6d9e16076aec017.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/4783b06aa8bdb5a97f09fdfb5b6167aaa83d157d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/51dcf9b20d9da4d5d412cbc09f466a6692f0aaad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/8bcb792f7eb9af6144aad2d67443c7682df0facb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/a415cf5ef28e106b59c29a45c91993610866cb86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/cf53978cea3f9152a63086bb40ed23dd613deda7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/d7ac8af8c4e9027e9106dd6bd48ab470e5c4d374.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/db9ac04b998095ecf47a36852b0a28f5caa17bc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817249.26/warc/dd7b9e913ef87737bb207cb71f7ad3e1037fd52a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/0b749df805de0f7051edbb3b6527b338374f3f79.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/0e96a60db6f9aff75972b0a09ab07769056535e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/210d435d6f0a0e8678770bf402d4a6784cbe77c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/262defabb7651469bb80cae245c41268810ae1cf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/63d68477ba910cd2c32768496a70cb9346c07051.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/a5d90ee10ac2fee3502118503eca6341412dff23.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/d6dba32999765540f6a59fd0dbac09ae7a62ce52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/dc69ea7ab1a1c0556e9480cd0a11630ca40e1617.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817253.5/warc/e5c59c556d55e674411091d812fd50d5e1ed7003.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/0482295272a1dd8c313931543a241c14eac781f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/19c6ea4929406398deb53a737bfa01f3f8ae93d9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/1d19719b7a59c5b9d00d122af111457695064914.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/2762da58e384117f2cd888515e1f527057d8d1d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/65a148ef09552fae326ad2a6e2fddf4a4da0b3fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/7273efce2550c92751dcb09da9dc96f3b7f9f04e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/8ab599b71232e4f6fc8d71c327866c728fa95dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/9b1bfc5b8e35d12df5e878a3c44bd1cc85fcda52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817289.27/warc/c2421f34d4546bc3261f96f8c14b31365f378744.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/35b3b2b77bd19ffbc41971fd59247cb744bee686.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/427b73bd81e8e3558e7b066ff4bac85df1481205.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/59fcc90ad0cc01db6b8a5f3f3cc95df0eba58abf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/7d32ff843b8129f2dff5daceccc0531915fe61b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/ae6b7158baf04e196c9b0b3aec3b089f726613a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/c16fc5a7c4ca06b4e96f3ab9eb80928b3ce1b342.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/c407b9835f76f946fd4ef3c8d557e5602c54779f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/ed72c3b19c223e4e030f60a4932b4d9b90eaf9e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817382.50/warc/f24ee6aedbf85e985c2fb75cbd4f96fda94213d3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/02bc8e5a67fa969d2804464d57893692bffa9e96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/16f22f4693b671e5ffef1f70983c22fd8de643ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/76bcc4b3363934d66e216a6710da49486bdac620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/7ff7db72b7bf7c8172bd7d4059c85f1c4d56b620.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/93749c0994ab6528f926e71fb0a183469db799f1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/98a9ca645be0923350a709e6adbec965b281f736.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/a579bca0106b5288543673bbf91b21167b3ff469.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/aa884a7c416c531bd6f738f586a841f183640441.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817398.21/warc/fba9673f7782e76cba945cba54b04aa8174e759d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/18f6c2679ea0f9c1eccd821192a9ae8ff0ad8dd4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/3e0f3b2ec8b538475f98534442361b42b4ee2e83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/43476f5da9bdc518aedc6cd80be22f44c3cb65dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/50ffdfd9275249a1c16b95f77ff563bb5ea55324.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/8dc00b8b9c556d210215e559409fa1909b210421.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/c082a3c88748ca89ddea4a19bdf3d11862ec045a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/c8e92da7395afe3293bee7405316308f2cdbb8b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/d9ff2f886a58aab5bae8c75961fdc971010d46fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817438.43/warc/dbff06faaee35f487afc37c07bab1eb9b052c65f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/02f66afa1c8f7e5553fd37a98a51513d09381bab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/15d463c92b6eaed458190bd89172e5cda5754835.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/6b6eaa34328aa4d9251ad2b199dd695595dc911a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/83157d64fe187a4b060cf3a3a3e128630e05708a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/a426804b08cdb85c757af8629d0d649201ae2a08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/a8e7138d8fa7f969c2a97e26c02a0592923210a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/b10347d7b0bd3043f32a829333f1f072765e02d4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/d4ff57b55b8f41d64a56c721e2f5217142f861f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817442.65/warc/e9c9fdf8ea2d8074edadb152ff1b1b649362f468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/4b40ec5cc7048462474b6a7cf175a84fa240aa27.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/5429a9f11c77249b2808285f0855b4b31c13fed4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/566ae05bd6a44edf80eff1c57b5eefac1805c1fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/814f92b4048b2d84c34d43a0b9566498bf8d7da6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/86dc8574e1f3c709a6d0c34dcabaac4138bfffc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/be545e657eac28b2e74810cb06436db9abcce447.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/e869be7161d492b9b17d0d48b9296c25dea4b24c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/edb04b403e6c4a43f972f8b7c567ea1981463c46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817455.17/warc/f85c257e6fb1eea8ddad25a95132aaa1f6d91b9d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/08c8e34a34d698306a85ae41b7d593beed01cccc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/0904f58dd6fb75867e9ddbdfb66be3229c00f097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/0d05fc12dbd9b26e2ec476e534512028b4d7c4c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/52082be600e49f9313ddb81929ea318fa2b8849c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/7580eea9865814c686b8fabb394b49503f4ca60d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/acda5cbff647d37d6eb81be84430a2ee7a6905c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/bf3a627bc6a6dda295a8dfcf6f3b1ebea46ef095.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/c4df6706ac0a89ba9864c793e88cce6aa2ffa2e0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817463.60/warc/dbfe7614b96f983f50ae8f8fb269f3675e92cd9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/2041532fc65e993d0afd45481ed5a402eb7e94a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/21d7c718e39b359fc96813fb7a30bb3eea80c51f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/261f95a9ad6a9cab068a777d8c7db806f49e7944.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/3b049a971bb2ddda8384a267efe902661ef35edc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/40a37e998d4594a87f7c5722f222fc45b4fbdcfe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/7e6fb8b5422e09cb0be7a707f2f76ee6287efbe6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/a15e47ed0ad68cb8274944e5caaf05fd7a530f25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/b558d2c7954a4445f0edca8f719e166ebe7a6cfd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817474.31/warc/ebb90939a3501fb1bdbdf41d1434bf0a63a388bd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/1003773a891610c9c6b849621f04ddd96bae0fa6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/6628909e1e4636b45797f88968682e6efde9c834.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/9529de1481445a2bc2d2748a23070a790ae982c4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/c8c5cf95db29a00d92d4a5eeca376433269bbbbf.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/cde99cdb00eca9432abdf5f91cc07c0448638e55.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/d519bf725fd8436d3b91a87a4dded28d0c7530ab.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/dd2135c3bc2c659dacd9bc8b3180fa99a3661f17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/f690e4118399e1974bf964e761a1dd157bcd912c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817491.77/warc/ff65c137ecbd6dc19094784bce0064700f056d0e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/36e671019f9221ebb296cf023a12db75a62e8d52.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/5027b3bb2d3e2be935a367556429dca59f751db1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/56aa15f45ccfb8312341621b93fc86fc6a08faa5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/6adbdeedb1018f63c92bfcb26ed721ded3664c28.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/71595b63b90086ba9e5f29086f55bea8300a2e77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/88c9d1d27cd236c26b68b81ce7950fd3e349981c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/8a8c1f08fdf76de76ff771f458b466463fb3eb8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/a57ff94ff41c13736f84e76cb2cba957e62ead9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817576.41/warc/b5b57df9143680f215c48f84db8b24bf46a3343d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/613fe79e612b9ef2a69d95d6160a972ef7a60556.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/7ea637befbb0ef0398dcfe5649777dd41e39d0c9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/8c356125b8db641aadac113f6038ec67116f5aed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/983f90150daea258b7d6cbbca7ad4fbbcdaa33a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/9860710c32fd6628715041f6eb364499463d9254.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/9a7bc55103e838677f2e7d2173fb52d3f8f412b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/b963231a1dd573c8661b9695717bce574205c92b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/cacbf9ed57913d1ab0575883f9d228ad940c5073.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817650.14/warc/f221635032b71d3e48f1824e30cd2fb641b7cf97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/038187424a0eaf57ba3e6f432d5a8c90bc0ff28b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/13ae730cf3f9e90f6b2f51674dd0753f4c16444a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/1d475e57f48d0fab8bd827bca93fb3d55bd5d654.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/85e38c00c6264e765d158b6eb961d46a8d5f712f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/900f88130b7793e902071f6e52a9b85d2908530e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/c748a3fed5d0d5b709166e98cd48f0ef1247f446.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/eac5f148aba5286021b603e5b434b438ac86ba02.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/efcc54536aecf62ee04b03838bc7366c94f91579.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817670.11/warc/f2e1c5c4cc270836292b1be8dfa2405620a53f34.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/0437f175e342ea79d784fa4511b00fc88b45fe16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/14b1334f97d9e28c58f0b5fcfc902bb9c46f09e8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/31caebd6b810d43b6d4018f0a3021c0005c82788.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/426070e167aac698a1005e1956ba3520e26f7d83.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/8569360e32504325920b2759265c1e24853ab30f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/a39f2df4dfffa5ff48766c8cdf29e766f015e685.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/a7a7a3c6b2b5b7c34313f0dbcc2183b08c5ee5f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/d7d67a64c9fed7fb67dbcfbef79f020950915d96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817674.12/warc/e6eb717ccd4d317520e45fa0936ea98d603ca73d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/15ee7af204fbc17185469873a465d787fe516baa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/530afec9a3c6c6344168a8e1e9f28b95faef3f64.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/583ce8af4544d91ec40be4ed5913530e58c65e11.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/8b52c4111475849cf4578276589eb92d9450f34e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/99c7bb4804592a52b21f5c0102d36aad42bffc2a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/a1c063ae0bc6eb1af9f1eb640381381511420d1c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/a5929cce0ef4aa35cb21622420cb2c5513597e0c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/aee86a9f7c6fc7b89c6aa8827ec5d633ad721aac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817688.24/warc/bdb906ae08d1e9ee022303da1fe2ef44f91a57c3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/110bc701b380b1fa9a6f6903a01e78805fe7c8ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/1de2854ee9b91a969b9a707cc8485ad01bea3ba5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/3baac23e8cec17a35bed4f95cadfc475adc50018.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/47e4a6c1dc3cb96a5c2cd4a72b707c65f07efa43.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/6516ad1d9e170a116777e8eaaccdea5b1717e8d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/929acfacc360de2c494ae3e1380542f902913ce9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/b8675713b33dc4fb897e517cfed1d2e1f07eb29d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/c2970f13f7e46c337e8da428442cd70e3f211765.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817699.6/warc/c64738704900564b5b4e853a9350dac25731edc2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/1ac2ca6e37d94626e38de61abaab910af2947459.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/20410875bd6d909939a41386db150cc65bdf939a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/25a0a0d4664f037b1dd2ac97a236ecc29736e275.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/5b8cf5c52d52fa8e97153262fc896174e9f946fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/5c27769636bc4bf6f2b1abcb502666fd24f7aaad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/6cfdaba69d1544080de67820f5cefb342032340c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/716d92222a959af959a9cce75778a3f8cc183efa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/89b4635a56753fe7187c5e62541444fe6227133c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.0/warc/94eb9103c07fb5edcdb165bd50f82cbfaccd97fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/0e2ca4a01aacfbc482f7ab906df66bd01acfb9db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/1f0c06d5b960860c90ef8397932e8bbf01c402ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/5d0b835a9202877d0bffffa18869e44c5a0805ae.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/8866c1093e1c3ff9916e4b18bdb30f56c2ea7eb7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/9cfc4b8bc5b3076360b34c6e4d6a9ce8ed3920f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/bf9fba4336ab355cf36511def1447af0d572581e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/da5d08807f09b4326a4ec875f2f6e58a43d6fb66.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/f8a05aeff962f5ddb03398bc3a7e9c0d15ea56cd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817729.87/warc/ffb044f17a18b2325191890d3ac5e4b53e07b8f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/08c5eae576197eda7507f24f0701ad544ee1a468.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/0d8b30e168063b3a1c67a33dd5ffb5e92cc1af17.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/3c0d8ac688764da3b3ba466e17965b7253bbc290.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/7f8cade2bb97794795fe7a40e69643e94b23f47d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/9972be7a1e2b9157d575efd241f6a8a1f2b1a9a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/9de9fd9fc1ae0816ba0c9ce393bd2d16ba0b38fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/cf979900e7e63826a31cbdf39a862461e57faf5a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/d572be0fe2c6cbf35f33ecdb6e4182369ba105de.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817765.59/warc/edcee41775c6b439f6e1e874ae1e34f7f8f9c9ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/1fd9d1689cc8c6822e3493c667f8f5105f3ccfc7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/35a93d4e7bc47242c206a950b9431f901f094c1e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/5830802a4d741def1503103392854162d1c9dcb2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/5b3981c20308e0fab177ff6c33e31f4cd1412ede.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/91dcc5773597960d67a5c43f0c6e48a965c19481.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/96b3461f210ec38e3226cbdefccf346dd1f8d173.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/a71e1a42a45e2ed59997d2b4d1344106eaf5c8a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/be703f39e7ab3e5fda68dd037c2fce22e6807141.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817780.88/warc/dd09d98d6717c8c4497ffa40010ab691173a5106.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/3619986d654374482044eef715f817fd5c06f24a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/67ee8faa1865b51684217b2224f901be7d0c0e46.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/7f8b3039eeb1855ec68cf5de731a06a37c0514f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/85ba3348e6ff5a6150bf4867f0e2eda07d3f36dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/8ebc383ab0b4951b2141dfb114dfa78606411ba4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/ad85ed8eede91e65817399d52d98d550a2da16fa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/ccf9253d333de91b07d283d2b48a3dd23e8480ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/de02a768adb374da7ff411f5b2755c13352becaa.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817790.98/warc/f2b5f1bd230364410fcfe857499fb63d8464cad4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/16102bffd1106ca5e469ec4bd1eb85ef52ca16a8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/569911f41b8885ac6b089b52fa0962d83df6e8c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/94de24d61d664ecbe17e3f1a293b893138c4dd7b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/b78467f6d939746e25d346732e699fe424b5d267.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/cd2d9dce9a5c5728b9a57771390581bf395cec9b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/ced5d7713fa5f9c97181bd7b856339c49ee2451e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/dd532d64365885552cefb85f0c10e399dcadd472.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/f1461ed1acdea322fe47730d3cc63f1b87363bb0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296817819.93/warc/f8a7630fbd8cf15ba48ce1f4728a4fa69021098f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/1977842bb3c034c16acfc679f8b3a1b8a2fbf924.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/218a3e6f753d642e2524cd2491d1ee846074efd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/3cb599d9cc53bd9cdbaf30377629ce1a49233e60.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/4a4616eb85b9f36b7ac5227080a91a3fd213d514.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/9d180636b755c8b7d0a119bff6ed4656fa512bd6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/9f7835183fb17c63788406de196052d5006a8e78.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/a0772e0b2396493ce8a41559b76b6a6e98d6550d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/bfb5a0fffd677dcbce31accc8255b15b799387a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818067.32/warc/c033250e24f21de1cfdead8a0d2a398fcac8255d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/0136329a14478531c8c67b078b513d28982076a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/040fd54e898dce9cfe17164ebec33e640a803954.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/15e5b18b7cc5336b3f022ddc81a3889512647f9a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/1c3815eaf7ef20118fdca01bf4b3ad45703fc631.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/642aaeb981c032dbad58c70cf262441951e52cce.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/89f67e7ab6c32f4a9a93c9a80d4d8fc4a33e4d67.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/b725e36819c428b8d09cb1221584208765fcceb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/d00fa9fa33363ab5bd421ebca5efc3251fcb6ac7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818072.58/warc/d698631fe8c493a959b899acd267ff154fec2e6a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/23acfb022756d703ca65fcf8e88139ec4ab7c17b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/2fb651ddf0f97f8d7a95a315e2ecb42371e7c682.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/62d9dcaef139c3703000c76349202af9b11efac0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/b9f980574e615c0e322cf5b337f3f07181f96d91.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/bdd4c7268adc7210a4ee738654927141ae0aab9e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/dc64944babb485053b8a5db0463aa95285edf510.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/eaca8fb29905d40d2852e5edbb368c5e37e85790.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/ebe82e98c667c92811466ffa7c154846c7bab313.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818081.81/warc/ff8f570061bd21124b13dd621832c4525a42c3ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/01ab523503aae4e4ee888e91a5e369f5f7ced90d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/1d692a2d57586377ea3543ab65139722420689c7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/3e2c4ce3254010c10e59f94d2f5978047c18330e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/57a27e46249f1f85f11873b4c60c8f018d722f80.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/722ab2a0a97b17cd215fac47be3339f1b8b54214.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/9a177b0c6800b6a98a50e6cd8366b31be19f910f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/c8f109fbcd1de7d101fff7f2e912b52d65b1d1c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/e2949f81092b3e6f172a7fee45101b59fed9f1f0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818105.48/warc/e3b7bb2db18cf7ba140646ae4df9b6203138e391.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/14049aeec1bfc7621fe26977653b8b169f5a065c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/6d8027ed61ca1b1b419948c57fca18f4a6d3ef99.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/852a5112215a3b6518db674c153feecb0a37ad21.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/86627f1f7136ee72cd9c1d5cc659d51282d19e5f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/aaaaeb58ba017c1e245ee94530c23366e03753ec.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/b7ac9989e6e08eb8e4690dc65cac64f8bd78a274.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/bdf0697252833bfbbbe88f1a8f28ffc07c29e9df.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/e04654895bf6eedbba4e731010dfe48bd2e1a783.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818293.64/warc/fc33f0200305bdcda5bf8d4362707d1df895bb81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/099ba6f5ff434b61d6f63af26b8bf8528a0109cb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/617dd5b740f9f793fde6e0b2a109e213eb0f58a1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/6c4fb90d79b7729b45d174582e454e0c44aeef86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/94360f57ee5fc19874d380385081ebceb78a4943.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/a0bdddd23d518b8659a8dd728ae358fbe0360e8d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/a462ad768b699330d47a32d11baa1d7304fb9097.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/b57d2335ab4d37e7c31c14e6e7457fd65be60721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/ddde3f56e7d951c4f7ae3c7ec5a9f5e569bc03ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818312.80/warc/f2302b121b006f542ce6ca36c835f4cf1c26f135.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/36f7045f24c43fcd3eeb02e31759b45825d64f85.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/378111bd1985cc78cba12fce3ea5fd822d70ac81.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/4d1f650585e765d04b404902fdc4551bee610298.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/6c8ccbff3e411492914300c77b24b4a51cfba41d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/6e6cc9557d7c09ae0bc6cd24a627725366d50989.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/7e59c91ec4377de4ccdeff522c41f3d8a242ee5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/cead11b7b863371abd40aa6a818ea22407fb02b2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/dfb9d64b908dd950a619ff87a8bf3bafaf09faee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818337.62/warc/fd943ba19186349c398e2c425bc146ad364eebee.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/03baa0255ffc9cf696c80e132338c2b80fa762dc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/127f4de74dee9f0f0cb537b3f38d1eb3c0a5f9b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/15c9385edbe98773e7cc5b1acb31fa74b0edd884.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/2f4dcf19874c939629e000dfb224e0c87f110786.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/61e84439cb966660e3ca773fcd3944195f50cf0a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/6ae9339f701dc375eb06fd005e7efd2808d6e009.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/6bf33aba4c67078fb246d1ad7e6c1e7fa257e8d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/b67b05dad64266b21bb22f3231d6475dbcfa7b8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818374.84/warc/ebe8cfc5055da65768c91374a5ef08a46bec0d88.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/405b4c61db92baca3db34386352aef9680ccce6d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/62daccc3e38fcd12aa97f0d5efee5d9d7788a59e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/82511ee5a1994eedecd058d55b40fa3db866cf5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/84729c46db4e163d63b1d5b7bcda41a40e564618.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/86d68751b522b4b26fee5de0caabcbb173b0070d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/879fdad1b06a9059bf7cc9e85eaec7308dc10a77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/adacf886c16092a307c63cde849d46d9fa15407d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/c9bf8c7acb13bb2060b0ad164bbbe3d0388e5838.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818452.78/warc/f2ec8758960df7b57db42e5bfc162fbbfb771125.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/791ee78ec34c2d648a2d08dd13f9718697f23282.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/82ee0d1c6a2a789c7c0d430868db47e105653360.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/979e353e01a893921dc85db0a1e84007689ed68c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/9ba70104f4aa4148a30a5eb2c5b4c3062460627f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/a01455bda2d4a8dea3aae0ed81f9eaccba0f23fe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/a9613e67a21ce41285900e81622c9cb94ad43a5b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/aae2772e631475dd302b9012180dfa867351a5ca.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/d11308f94ba150acc721c4a4192d8382a181e9e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818464.67/warc/fd3df5ab6af848bf376a6dfa6241ad754ecc635f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/10a0371b2cd86f594660e715b5b1ea0213ad2d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/203c430fae57ef5ce4cc930b535a7d2c9950bfb4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/239288a183e6752933b890b3ea21ea513b4bf2fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/50696e58b89d1b8ab322fe797f92f8cd51adadc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/7a013d68e08e591f8b1ffaaa831285a133c608b3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/86f7fb3c4bfa795392ce6f461b6094caf1c9fa3e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/b07fd868198d5c780944fb9a14c2516190223d5e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/c2d851a0202093320457ccae6fe5872cf5cc2fa8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818468.34/warc/c91ca1c1e0293580b7f6e91b277efc051090b8d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/10d745158d31e269ad497d8a66bd5563eb14b9e6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/2b2e034e2d88db1dacb007ecda5d67eba1c195a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/4c68f19b64f8202f76df3f447fdc5fec21a9dabc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/7067e1533d9a25fbcfb12e22365c39bf2a48a3a0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/77f5e83226f952a582b09589065436ef2daa988c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/85671859f751d21b8e48fae8afae9705ad9481b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/c5536f8f6575ed7d2117609c283dee528246d516.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/e50c6bf2ce86422f3ddfae9611c390aff6ae757a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818474.95/warc/ea75e7bdec31ad6d4ac93e2cfcc28ef47b2794ef.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/281ede9859f31168a021cff2faac49e8d8ce290b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/2c2baf9fab757dea79b91141ccd754735b62871f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/2ce506d24386d5778c4fdb84ddeac67940294485.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/4af49fc4385b59bd30559018e2c047dd3fd85f0b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/67d2ba1ce58cbbf32402213cd80db0b867c1834e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/9f706008c571b9ef279effa8454dfd4ef1c073b0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/b2b59ddd05288f17633c5c61521d22b6b5de8d41.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/d8fd4c73c960db5a74812dec9fc45c17f66fe041.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818711.23/warc/f6d97096d0c9a54e4732c7861743dae0e20ff7a5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/1ab066f6622034eb08ee5f188878494c1f2461a4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/38c35ecaf4155db8155d573f5dce9b1baa2469a3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/501472904b43a25606cd256ae2eb1133c4824102.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/66fdbfcb1910baf6270bc53507eb5726cf0a6378.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/a605c9b847eaacb4d6cb8a492328f32aa056435a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/ab8547655543a7522cb3c95fa80f2c51572434c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/e62657fd7d44fbb2534d0f1ed9260cf0e16186b5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/f06de0228d3dee69b876e1fdf1e470509fa74035.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818732.46/warc/f7b1c449f2a8cd5abff8aa7716301f9f27a36053.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/03961c2360a360e1ad71e176a1f09ad0421fe2fb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/209b2bd62a92e2cc97cdb9c603d061e6490406a6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/2939f4f0bc207c58b4b90250ba9904c5d9e6a897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/4e16843d39752e6094ebb1e056a9d4c3092638f6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/7d98e27122ee9c200f13852d520c95f9e0288306.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/97ae59edd64d4b075d2ff9fe8d87d285c179e0f7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/c51ed7331899857d4d9cd7aabb16794ca5acf3fd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/c9091ca35414b6ada89faa9ab688e7e0fc5ecfc8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818740.13/warc/d841617d75cf1960d6e7445003ad46e5c13ed878.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/16673ac1f9ccfd11d3b4825b489ff90018ff3a09.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/2c009c248f2f9533d14f2a908aea929622da1f4d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/2f1ca563b11a304959e447aea63cd7dca73cde84.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/82538d1d0536174596ea6bf20de74c646c2946f4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/9ddd527fece21d5f3848b297dd8b1f61e7810b97.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/a4ce277a7b4d9bf4674b45a6c9a52caf28c654e5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/c4f0d2e0e696e65b3b8622031e1a132b49604fbe.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/ec6318b85f033df5c2c922c7e1c0bd4d1c6d8609.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818835.29/warc/f17a85cd7454d2aa56a93d64c0662c3751aa99a7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/01b571981d1974551794268d3002832a0bd11721.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/1b20ceb3dbc721d013f86ce65100f33666cd9deb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/285ab6371e4fee521b9aebdf7f420a3035007167.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/2a88edeb144f6f41b7490e14190ffea23267b971.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/321f0c4a81c573b66cd913637cf246fdc4ffab86.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/709798c102c73fb85412ff6fcb5cf39fa067bdd5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/7a5fad39ddc66ecfd8a1d6dc7d9e7574aab6bc0f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/80a05b13ee7e1da15f115ad6d9be9bd482bf6692.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296818999.68/warc/ba9e87ff7791bf981bbf2ae093eb268e58cb236f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/2027d883f6f58f22f0a527dbf2ebea4e86dac113.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/276aa2181d448147ac1b2d21cf8201bdbacf4812.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/27933e1c331de2b9bb8ca3a9958787f600ce07f3.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/2f0c2331cbd2da0023a8188a0005ed440e07a0e2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/3d25400c1db1acb67622f7a479c73fc31e22f2e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/7fcb86be4c2565be87f65e5d62b9f3fd7aa5c17e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/c64d60d42c7b5c0ea643a00f39a5096e20f9aa08.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/f3536aaac53d58075e63fd69855be871cfd3a65e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819067.85/warc/f4adde80fe23ee8dd541447b6f474c604978163b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/012dd021f6cbd2b55283c692813e833557ad4d25.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/13f2800b368ccb6abf06fbb096f2042c1a3842f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/31dfe112302292d6d4bda9907ae17fd58270975d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/329901f77dde630994c7e41a9bd45ca9d8d15fe0.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/51a2010cbe8503d8f048c28a2ec35341268ad2c2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/79d539e0f471e2854b5a905ec18076c929cf7055.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/7fef9a026ecd4b4c53e5e5ea0a91e84d4b07574f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/8a5211efa47c10c39aca5a26df45b9f1e4e66d3d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819089.82/warc/d8e5d12df59dbc4f0e5e9dccf9cf53e79aa940bb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/156d9ffb0007f8be0454cd3e44f94ab487c93f8f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/275ebda0215930db0466d9cedbb96c2c935a8781.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/294198e139487b798a0c0a207d717d3abe890353.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/3e40f312fcf40179f3a7ed32213732c5a9499433.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/410e33ee9a371696bd1786db48e4880a72d86207.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/8b9fc65f7df4a7ed8c9043feaed966c03c18a4e4.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/c7e12bed8237674c4b39850ae904159b823a0897.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/cf90fab17055a85bf61a17a938b227b415ea9adc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819273.90/warc/fa2051e01ca9899035fd8cffefca95f205c2b022.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/0a8289f6fff3fbcb4659be1c3f988e3f0c54bb96.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/0e8ec3499e7cd000347860eb7866310f67abeb77.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/2429e28062385756c24f69e8e40b564bcd1d26ad.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/2f9c3f9638f3abb5da7b1bd8df4897da46718ed9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/355bc717c616847e494ca030bbae1fb421c240f8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/3762ff1e573d2c2300cbf9a18b6379d2f69d478f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/40d635796e872f697e09470b5326240b55e1b7ed.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/8b25b641306b57ed612f99411929963b077e95db.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819668.74/warc/8b354005c35a59bf8e9af09ffcac881c708c15b7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/49d50e3c23fd9460a833defe564d66917f262dfb.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/52c807d1cd436134d6eb889085e7a124fd1c11dd.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/740f9a8c3ba8315626500274c9d75cf626f1ec98.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/9129d130d1aaa47963f19b2dd8b7c70cb34a2713.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/99bddb9f924b646573ac35aa66fe5a3358c343b9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/bef52a1b637701b87a1a0c24d95fddba8a525a6c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/cea404136a47a966563034055c0e4209c42b6636.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/d5b5d96ac5109873ba5b89a3ede9ff19f079a439.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819847.83/warc/d79d653f5e2eeef4b471322d1d7b62740f3e2c01.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/147695464313109d94edf8625dc9f9aea65c9d16.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/57b490f1a03157bf1f1f729e6c064c70118f695d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/886f95704cae45f946894539f508c10b2775df4e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/9ff09bd01e2f4993e6e70a2012e77e8125427b4f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/b64179155e845235f195de5e15e65ef8b606f464.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/d266b37e2e14cfc2ab0b37850b8295e433cec997.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/ebe2224da917de966a8cb7dc4e86c5e642adb0d6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/f22fb5659ce4bd607020ee7491d803040f46e490.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296819971.86/warc/f57dbae6584a2c37d8b4570627345fd2ba55f872.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/337104b9c82021dd8a3097e5fccfe286e0fb95fc.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/36c1b62528e764b891dff3051ce4462e03b0724d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/3cc3a8a47667c4084f7d7d5d036e567edc65109e.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/4bc59b5a0b264f18faeefc3c59932e6ba591377f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/95fcf92b4a1ed5e895cb89e96fcc88c15f5cc259.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/b227b34563d17b8073c33a2fa250fc681eb5e49b.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/c259bd9656a87838dc4d617bdb7550c0ca36cdac.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/d08f8b4a7425db4ce6fa7665b4adce4df8117150.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712296820065.92/warc/de826083ec2d8da743b8ead4201e0c20ad912474.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/0090ba7f30524145cb7a9e9d0b07745ec1a336c5.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/10992fc8038636939baf7ac3e4d00b3232e61536.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/2ab75f8e437dd18902fe2f3b075b2a39b2a759ea.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/4b70fea68167de131a2a1bce27ae0d535b24406c.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/4eb65b9d15132d319dc0eec54c63c88c7cba4c8a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/5cde2995deaf4ebdf4ab0f3177cab71a3ce602f2.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/8de391b089eccd25f4a884cb6ea15e39c89b74e9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/b6ec05e93e0e389d65f1f70259966e2d844e92d7.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297284704.94/warc/fc8e2cc1e267fbef08e3dd1ba4dc27b9d6138794.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/0d82d5d2841c3cf75cc8ce1e95e2e3fcff597842.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/13fdd8ef13b370d79821ea1110b8734371f61ad1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/18d829d23145bb3cf6109464bf372f028703d225.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/799ddc31b69a410cde689e09a6d21fc87d9f18c6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/8176dfa55a2c53ccee09e99f3dd7802f4661100d.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/8bdca58aa641a2cb755a744c0b2bb2f09811d1a9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/9309c8d6a2d90d9a691035801d73abfc2a0dfca8.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/a4c895e865c207d4ecadfb2d2af4850569a7b7e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297290384.96/warc/c1f7ca5447935587b848bc376694b503c3a3db48.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/1f0d985387917aeb49a8888e3f9868cb2133a3e1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/22a7d28ab65c303059080a2ab592c5795551228a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/50ab8f9a85c1fb009d856acd38e8a5c2c04178f9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/6ca57704cd33c52b060254a45db6250cf69e95ba.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/6e46319f46037a0701ca4de1517e3cc14b118031.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/708a9f026019503895096477ecde65039c36184f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/7547b0bddde3b749fe7b14de4650d89abdbbde3a.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/c128b91e1b138d023a3329dc2d3367fe2e7074c1.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297292879.97/warc/f1e30379be3473ea6513063930a86d8ca6d48256.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/07513acfca69f00eb64a7535098265c8b21caf40.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/23ffe98038c2adc062ffc9f973130b36e68e41b6.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/374d2cc43c7ac1ed2a7c40d89d089e987f96bc56.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/3dd602b154f343aeed42d636b5bd123f0da3e41f.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/5c01e00f67ad42c5979adc032055242ee3185155.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/5cda7407ea2475fc946527b451bc27df1517c443.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/654e8ce5892b86980abdae1b9e2b3ee947525cc9.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/d2687d8bcc3eaae4953561699a3f98cf20a82710.jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/1712297295329.99/warc/fc647730393344cdabc5b347e164ae16907602cd.jsonl.zst + attributes: + - c4_v2 + - dedupe_url + - ft_lang_id_1e2 + - gopher_v2 + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 2_000_000_000 + path: s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2024-18 + min_text_length: 25 # matches wikipedia + filter: + syntax: jq + include: + # Only English + - >- + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en != null) and + (.attributes.ft_lang_id_1e2__ft_lang_id_1e2__en[0][2] > 0.5) + exclude: + # Duplicated URLs + - (.attributes.dedupe_url | length > 1) + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] < 50) + - >- + (.attributes.gopher_v2__gopher_v2__word_count != null) and + (.attributes.gopher_v2__gopher_v2__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v2__gopher_v2__median_word_length != null) and + (.attributes.gopher_v2__gopher_v2__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio != null) and + (.attributes.gopher_v2__gopher_v2__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v2__gopher_v2__required_word_count != null) and + (.attributes.gopher_v2__gopher_v2__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v2__gopher_v2__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Remove NC or ND licensed pages + - >- + (.metadata.attribute_spans != null) and + (.metadata.attribute_spans | keys | map(select(test("_nc_"|"_nd_"))) | length > 0) + +work_dir: + input: "/tmp/cccc-CC-MAIN-2024-18/mix/input" + output: "/tmp/cccc-CC-MAIN-2024-18/mix/output" + +processes: 100 diff --git a/configs/cccc/mixer_test.yaml b/configs/cccc/mixer_test.yaml new file mode 100644 index 00000000..edaba9f0 --- /dev/null +++ b/configs/cccc/mixer_test.yaml @@ -0,0 +1,43 @@ + +# streams: +# - name: cccc-CC-MAIN-2022-05 +# documents: +# - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/*.gz +# attributes: +# - dedupe_doc +# - dedupe_para +# output: +# max_size_in_bytes: 2_000_000_000 +# path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/cccc/v2/documents/CC-MAIN-2022-05 +# filter: +# syntax: jq +# exclude: +# # Exact duplicates +# - (.attributes.dedupe_doc != null) + +# # Fuzzy duplicates with above 0.5 of shared ngrams (TO CHANGE) +# - >- +# (.attributes.dedupe_para | length == 0) or +# ((.attributes.dedupe_para | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.5) + +# processes: 100 + +streams: + - name: cccc-CC-MAIN-2022-05 + documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v1_gopher_fix/documents/CC-MAIN-2022-05/*.gz + attributes: + - dedupe_para_test + output: + max_size_in_bytes: 2_000_000_000 + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/cccc/v2-test/documents/CC-MAIN-2022-05 + filter: + syntax: jq + include: + - (.attributes.dedupe_para_test | length > 0) + +# work_dir: +# input: "/tmp/cccc-CC-MAIN-2022-05/mix/input" +# output: "/tmp/cccc-CC-MAIN-2022-05/mix/output" + +processes: 100 diff --git a/configs/cccc/run-dedupe-doc-step-1.sh b/configs/cccc/run-dedupe-doc-step-1.sh new file mode 100644 index 00000000..97c3f6e7 --- /dev/null +++ b/configs/cccc/run-dedupe-doc-step-1.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2014-42.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2014-49.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2016-07.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2016-18.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2016-22.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2017-09.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2017-22.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2017-26.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2018-13.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2018-22.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2018-26.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2018-30.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2019-04.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2019-09.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2019-13.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2019-35.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2019-39.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2020-29.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2020-34.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2021-17.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2021-43.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2021-49.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2022-05.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2023-06.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2023-14.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2023-23.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2023-50.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2024-10.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2024-18.yaml dedupe \ No newline at end of file diff --git a/configs/cccc/run-dedupe-doc-step-2.sh b/configs/cccc/run-dedupe-doc-step-2.sh new file mode 100644 index 00000000..ecaf66e9 --- /dev/null +++ b/configs/cccc/run-dedupe-doc-step-2.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2013-20.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2013-48.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2014-10.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2014-15.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2014-23.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2014-35.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2014-41.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2014-52.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2015-06.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2015-11.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2015-14.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2015-18.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2015-22.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2015-27.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2017-04.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2017-13.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2017-17.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2017-51.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2018-34.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2018-47.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2018-51.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2019-30.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2021-39.yaml dedupe \ No newline at end of file diff --git a/configs/cccc/run-dedupe-doc-step-3.sh b/configs/cccc/run-dedupe-doc-step-3.sh new file mode 100644 index 00000000..a74fcaf0 --- /dev/null +++ b/configs/cccc/run-dedupe-doc-step-3.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2016-26.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2016-30.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2017-30.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2018-09.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2019-43.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2019-51.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2020-10.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2020-24.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2020-40.yaml dedupe +dolma -c configs/cccc/dedupe-doc/CC-MAIN-2023-40.yaml dedupe \ No newline at end of file diff --git a/configs/cccc/run-dedupe-para-step-1.5.sh b/configs/cccc/run-dedupe-para-step-1.5.sh new file mode 100644 index 00000000..3a047875 --- /dev/null +++ b/configs/cccc/run-dedupe-para-step-1.5.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -ex + + +dolma -c configs/cccc/dedupe-para/CC-MAIN-2014-42.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2014-49.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2016-07.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2016-18.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2017-09.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2017-22.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2018-13.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2018-22.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2019-04.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2019-39.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2021-43.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2023-06.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2023-23.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2023-50.yaml dedupe diff --git a/configs/cccc/run-dedupe-para-step-1.sh b/configs/cccc/run-dedupe-para-step-1.sh new file mode 100644 index 00000000..a004ce45 --- /dev/null +++ b/configs/cccc/run-dedupe-para-step-1.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/dedupe-para/CC-MAIN-2014-42.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2014-49.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2016-07.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2016-18.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2016-22.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2017-09.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2017-22.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2017-26.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2018-13.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2018-22.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2018-26.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2018-30.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2019-04.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2019-09.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2019-13.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2019-35.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2019-39.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2020-29.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2020-34.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2021-17.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2021-43.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2021-49.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2022-05.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2023-06.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2023-14.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2023-23.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2023-50.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2024-10.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2024-18.yaml dedupe \ No newline at end of file diff --git a/configs/cccc/run-dedupe-para-step-2.sh b/configs/cccc/run-dedupe-para-step-2.sh new file mode 100644 index 00000000..628cc847 --- /dev/null +++ b/configs/cccc/run-dedupe-para-step-2.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/dedupe-para/CC-MAIN-2013-20.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2013-48.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2014-10.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2014-15.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2014-23.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2014-35.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2014-41.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2014-52.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2015-06.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2015-11.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2015-14.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2015-18.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2015-22.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2015-27.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2017-04.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2017-13.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2017-17.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2017-51.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2018-34.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2018-47.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2018-51.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2019-30.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2021-39.yaml dedupe \ No newline at end of file diff --git a/configs/cccc/run-dedupe-para-step-3.sh b/configs/cccc/run-dedupe-para-step-3.sh new file mode 100644 index 00000000..ce7c4897 --- /dev/null +++ b/configs/cccc/run-dedupe-para-step-3.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/dedupe-para/CC-MAIN-2016-26.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2016-30.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2017-30.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2018-09.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2019-43.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2019-51.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2020-10.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2020-24.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2020-40.yaml dedupe +dolma -c configs/cccc/dedupe-para/CC-MAIN-2023-40.yaml dedupe \ No newline at end of file diff --git a/configs/cccc/run-dedupe-url-v2.sh b/configs/cccc/run-dedupe-url-v2.sh new file mode 100644 index 00000000..e36bcd6f --- /dev/null +++ b/configs/cccc/run-dedupe-url-v2.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +set -ex + diff --git a/configs/cccc/run-dedupe-url.sh b/configs/cccc/run-dedupe-url.sh new file mode 100644 index 00000000..d85cd7ed --- /dev/null +++ b/configs/cccc/run-dedupe-url.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/dedupe-url/CC-MAIN-2013-20.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2013-48.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2014-10.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2014-15.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2014-23.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2014-35.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2014-41.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2014-42.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2014-49.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2014-52.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2015-06.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2015-11.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2015-14.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2015-18.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2015-22.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2015-27.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2016-07.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2016-18.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2016-22.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2017-04.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2017-09.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2017-13.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2017-17.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2017-22.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2017-26.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2017-51.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2018-13.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2018-22.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2018-26.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2018-30.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2018-34.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2018-47.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2018-51.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2019-04.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2019-09.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2019-13.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2019-30.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2019-35.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2019-39.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2020-29.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2020-34.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2021-17.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2021-39.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2021-43.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2021-49.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2022-05.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2023-06.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2023-14.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2023-23.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2023-50.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2024-10.yaml dedupe +dolma -c configs/cccc/dedupe-url/CC-MAIN-2024-18.yaml dedupe \ No newline at end of file diff --git a/configs/cccc/run-mix-part-1.sh b/configs/cccc/run-mix-part-1.sh new file mode 100644 index 00000000..dae34f07 --- /dev/null +++ b/configs/cccc/run-mix-part-1.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/mixer/CC-MAIN-2019-35.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2019-09.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2022-05.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2021-17.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2021-49.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2024-18.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2023-14.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2020-29.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2020-34.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2016-22.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2024-10.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2018-30.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2019-13.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2017-26.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2018-26.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2021-43.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2017-51.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2019-39.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2023-50.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2018-13.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2016-18.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2016-07.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2023-23.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2018-22.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2019-04.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2017-22.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2023-06.yaml mix diff --git a/configs/cccc/run-mix-part-2.sh b/configs/cccc/run-mix-part-2.sh new file mode 100644 index 00000000..9cff10e4 --- /dev/null +++ b/configs/cccc/run-mix-part-2.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/mixer/CC-MAIN-2013-48.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2014-23.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2017-13.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2017-17.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2015-27.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2014-52.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2015-11.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2015-18.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2015-14.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2014-41.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2014-10.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2018-47.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2014-49.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2015-22.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2014-42.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2017-04.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2013-20.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2014-15.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2018-34.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2019-30.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2015-06.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2014-35.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2021-39.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2017-09.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2018-51.yaml mix \ No newline at end of file diff --git a/configs/cccc/run-mix-v2-step-1.sh b/configs/cccc/run-mix-v2-step-1.sh new file mode 100644 index 00000000..9371db2d --- /dev/null +++ b/configs/cccc/run-mix-v2-step-1.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/mix-v2/CC-MAIN-2014-42.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2014-49.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2016-07.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2016-18.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2016-22.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2017-09.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2017-22.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2017-26.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2018-13.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2018-22.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2018-26.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2018-30.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2019-04.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2019-09.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2019-13.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2019-35.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2019-39.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2020-29.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2020-34.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2021-17.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2021-43.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2021-49.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2022-05.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2023-06.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2023-14.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2023-23.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2023-50.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2024-10.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2024-18.yaml mix \ No newline at end of file diff --git a/configs/cccc/run-mix-v2-step-2.sh b/configs/cccc/run-mix-v2-step-2.sh new file mode 100644 index 00000000..3b4880cd --- /dev/null +++ b/configs/cccc/run-mix-v2-step-2.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/mix-v2/CC-MAIN-2013-20.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2013-48.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2014-10.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2014-15.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2014-23.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2014-35.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2014-41.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2014-52.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2015-06.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2015-11.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2015-14.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2015-18.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2015-22.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2015-27.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2017-04.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2017-13.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2017-17.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2017-51.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2018-34.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2018-47.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2018-51.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2019-30.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2021-39.yaml mix \ No newline at end of file diff --git a/configs/cccc/run-mix-v2-step-3.sh b/configs/cccc/run-mix-v2-step-3.sh new file mode 100644 index 00000000..df29e84b --- /dev/null +++ b/configs/cccc/run-mix-v2-step-3.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/mix-v2/CC-MAIN-2016-26.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2016-30.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2017-30.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2018-09.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2019-43.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2019-51.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2020-10.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2020-24.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2020-40.yaml mix +dolma -c configs/cccc/mix-v2/CC-MAIN-2023-40.yaml mix \ No newline at end of file diff --git a/configs/cccc/run-mix.sh b/configs/cccc/run-mix.sh new file mode 100644 index 00000000..50cb7811 --- /dev/null +++ b/configs/cccc/run-mix.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -ex + +dolma -c configs/cccc/mixer/CC-MAIN-2023-14.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2023-06.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2018-26.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2016-18.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2019-35.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2020-29.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2017-22.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2019-13.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2017-26.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2018-30.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2021-43.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2021-17.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2016-22.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2018-22.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2024-10.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2016-07.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2024-18.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2020-34.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2019-39.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2019-09.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2019-04.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2017-51.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2023-23.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2023-50.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2018-13.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2022-05.yaml mix +dolma -c configs/cccc/mixer/CC-MAIN-2021-49.yaml mix \ No newline at end of file diff --git a/configs/cccc/stats.yaml b/configs/cccc/stats.yaml new file mode 100644 index 00000000..e6a8571d --- /dev/null +++ b/configs/cccc/stats.yaml @@ -0,0 +1,71 @@ +attributes: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-40/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-40/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-24/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-10/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-10/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-51/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-51/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-43/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-43/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-09/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-30/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-30/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-30/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-30/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-26/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-26/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/*.zst + +taggers: + - whitespace_tokenizer_v1 + - ft_lang_id_1e2 + - tokenizer_repetitions_v2r2 + - c4_v2 + - gopher_v1 + +processes: ${d.procs:} diff --git a/configs/cccc/taggers_part_1.yaml b/configs/cccc/taggers_part_1.yaml new file mode 100644 index 00000000..2dfcca9b --- /dev/null +++ b/configs/cccc/taggers_part_1.yaml @@ -0,0 +1,72 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-18/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2024-10/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-50/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-40/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-23/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-14/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2023-06/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2022-05/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-49/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-43/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-17/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-40/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-34/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-29/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-24/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-10/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2020-10/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-51/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-51/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-43/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-43/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-39/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-35/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-13/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-09/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-04/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-30/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-26/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-22/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-13/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-09/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-51/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-30/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-30/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-26/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-22/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-30/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-30/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-26/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-26/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-22/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-18/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2016-07/*.zst + +taggers: + # - whitespace_tokenizer_v1 + # - ft_lang_id_1e2 + # - tokenizer_repetitions_v2r2 + # - c4_v2 + # - gopher_v1 + - gopher_v2 + +processes: ${d.procs:} diff --git a/configs/cccc/taggers_part_2.yaml b/configs/cccc/taggers_part_2.yaml new file mode 100644 index 00000000..625866d8 --- /dev/null +++ b/configs/cccc/taggers_part_2.yaml @@ -0,0 +1,57 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-20/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2013-48/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-10/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-15/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-23/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-35/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-41/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-42/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-49/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2014-52/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-06/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-11/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-14/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-18/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-22/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2015-27/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-04/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-09/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-13/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2017-17/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-34/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-47/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2018-51/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2019-30/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/*/warc/*.zst + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/CC-MAIN-2021-39/*.zst + +taggers: + # - whitespace_tokenizer_v1 + # - ft_lang_id_1e2 + # - tokenizer_repetitions_v2r2 + # - c4_v2 + - gopher_v2 + +processes: ${d.procs:} diff --git a/configs/crawl/cccc.yaml b/configs/crawl/cccc.yaml new file mode 100644 index 00000000..5bf63eea --- /dev/null +++ b/configs/crawl/cccc.yaml @@ -0,0 +1,28 @@ +documents: ${d.stdin:} +destination: + - s3://ai2-llm/pretraining-data/sources/cccc/v0/documents/${oc.env:SNAPSHOT} +processes: ${d.procs:} +source_name: cccc_${oc.env:SNAPSHOT} +linearizer: resiliparse +pre: + taggers: + - cc_re + skip: true + +post: + taggers: + - copyright + skip: false + +store: + html: false + attr_spans: 500 + +skip_duplicate_urls: true +skip_checks: true + +batch_size: 100 + +work_dir: + input: ${oc.env:HOME}/cccc/${oc.env:SNAPSHOT}/input + output: ${oc.env:HOME}/cccc/${oc.env:SNAPSHOT}/output diff --git a/configs/crawl/ft2-mix_CC-MAIN-2024-26.yaml b/configs/crawl/ft2-mix_CC-MAIN-2024-26.yaml new file mode 100644 index 00000000..164892c5 --- /dev/null +++ b/configs/crawl/ft2-mix_CC-MAIN-2024-26.yaml @@ -0,0 +1,117 @@ +streams: + - name: cc-science-v2 + documents: + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861173.16/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861183.54/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861261.53/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861319.37/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861342.11/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861342.74/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861372.90/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861451.34/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861480.87/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861488.82/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861517.98/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861520.44/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861521.12/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861545.42/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861546.27/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861567.95/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861568.20/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861575.66/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861578.89/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861583.78/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861584.65/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861586.40/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861594.22/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861605.77/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861606.63/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861618.0/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861640.68/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861643.92/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861657.69/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861659.47/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861665.97/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861670.48/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861671.61/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861674.39/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861696.51/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861698.15/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861701.67/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861719.30/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861733.59/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861737.17/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861741.14/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861746.4/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861747.46/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861747.70/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861752.43/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861762.73/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861773.80/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861794.76/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861796.49/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861797.58/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861806.64/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861817.10/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861825.75/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861828.24/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861832.94/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861853.72/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861880.60/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861883.41/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861916.26/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861940.83/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861957.99/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198861989.79/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862006.96/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862032.71/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862036.35/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862040.7/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862070.5/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862125.45/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862132.50/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862157.88/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862189.36/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862249.29/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862252.86/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862310.25/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862396.85/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862404.32/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862410.56/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862420.91/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862425.28/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862430.93/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862464.38/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862466.81/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862474.84/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198862488.55/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198864850.31/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198864968.52/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198864986.57/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198865074.62/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198865348.3/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198865383.8/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198865401.1/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198865482.23/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198865490.6/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198865545.19/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198865560.33/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198865694.2/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198865972.21/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198866143.18/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198866218.13/warc/*.jsonl.gz + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/1718198866422.9/warc/*.jsonl.gz + + attributes: + - owmV2_FTsciV1_comb_vvhth_qt + + output: + max_size_in_bytes: 1_073_741_824 + path: s3://ai2-llm/pretraining-data/sources/cc-science/v2/documents/CC-MAIN-2024-26 + + filter: + include: + - .attributes.owmV2_FTsciV1_comb_vvhth_qt__owmV2_FTsciV1_comb_vvhth_qt__science != "null" + syntax: jq + +processes: 188 diff --git a/configs/crawl/ft2.yaml b/configs/crawl/ft2.yaml new file mode 100644 index 00000000..284750fd --- /dev/null +++ b/configs/crawl/ft2.yaml @@ -0,0 +1,19 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2019-04/*/warc/*jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2020-05/*/warc/*jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2021-43/*/warc/*jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2022-49/*/warc/*jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2023-06/*/warc/*jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2023-40/*/warc/*jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-18/*/warc/*jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-26/*/warc/*jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-30/*/warc/*jsonl.zst + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/CC-MAIN-2024-33/*/warc/*jsonl.zst + +taggers: + - owmV2_FTsciV1_comb_vvhth_qt + +processes: 1 # depending on memory usage of a single tagger, you might need to reduce this number + +# for processes, start with maybe 64 and check memory usage of a single tagger +# you can increase up to 188 if each tagger uses less than < 2GB diff --git a/configs/crawl/science-v1.yaml b/configs/crawl/science-v1.yaml new file mode 100644 index 00000000..afd84129 --- /dev/null +++ b/configs/crawl/science-v1.yaml @@ -0,0 +1,34 @@ +documents: "${d.sed:${d.split:${d.file:${d.cache:https://data.commoncrawl.org/crawl-data/${oc.env:SNAPSHOT_ID}/warc.paths.gz}}},crawl-data@s3://commoncrawl/crawl-data@,@}" # download list of warc files from Common Crawl (d.cache), read the file (d.file), split the file into lines (d.split), and add the bucket prefix by replacing the string "crawl-data" with "s3://commoncrawl/crawl-data" (d.sed) +destination: + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/${oc.env:SNAPSHOT_ID} # save the documents in the S3 path; note that you have to provide SNAPSHOT_ID as an environment variable +processes: ${d.procs:} # use the number of processors available on the machine; in practice, I override this value in the command line by using 40 processes on a machine with 370GB of RAM because each process consumes around 9GB of RAM +source_name: science_${oc.env:SNAPSHOT_ID} +fast_linearizer: fast-p-less-space # this is the fast linearizer +linearizer: openwebmath # this is the openwebmath linearizer (final text) + +min_text_length: 150 # minimum text length to keep the document + +pre: # these run before HTML linearization + taggers: + - owm_math_v2 # extract math tags from html documents + - owm_latex_v2 # extract latex tags from html documents + - science_kw_v2 # extract the science keywords from the documents + skip: true # if none of the three taggers above returns any tags, skip the document + +post: + taggers: + - owmV2_FTsciV1_comb_lth_qt # this is the tag that either looks for math/latex, or runs the fasttext science model (quantized) + - ft_dolma_doc_eng # language filter + skip: true # if the tagger above does not return any tags, skip the document + mode: all # filter out documents that do not have any tags + +skip_checks: true # this skips checking if the paths in `documents` exist +skip_duplicate_urls: true # this skips checking if the URLs in the documents are unique +batch_size: 10 # each process does 100 files at a time + +store: + html: false # do not keep the html in metadata + +work_dir: + input: ${oc.env:HOME}/progress-science-v1/${oc.env:SNAPSHOT_ID}/input + output: ${oc.env:HOME}/progress-science-v1/${oc.env:SNAPSHOT_ID}/output diff --git a/configs/crawl/science-v2.yaml b/configs/crawl/science-v2.yaml new file mode 100644 index 00000000..e37f2a3b --- /dev/null +++ b/configs/crawl/science-v2.yaml @@ -0,0 +1,34 @@ +documents: "${d.sed:${d.split:${d.file:${d.cache:https://data.commoncrawl.org/crawl-data/${oc.env:SNAPSHOT_ID}/warc.paths.gz}}},crawl-data@s3://commoncrawl/crawl-data@,@}" # download list of warc files from Common Crawl (d.cache), read the file (d.file), split the file into lines (d.split), and add the bucket prefix by replacing the string "crawl-data" with "s3://commoncrawl/crawl-data" (d.sed) +destination: + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/${oc.env:SNAPSHOT_ID} # save the documents in the S3 path; note that you have to provide SNAPSHOT_ID as an environment variable +processes: ${d.procs:} # use the number of processors available on the machine; in practice, I override this value in the command line by using 40 processes on a machine with 370GB of RAM because each process consumes around 9GB of RAM +source_name: science_${oc.env:SNAPSHOT_ID} +fast_linearizer: fast-p-less-space # this is the fast linearizer +linearizer: openwebmath # this is the openwebmath linearizer (final text) + +min_text_length: 150 # minimum text length to keep the document + +pre: # these run before HTML linearization + taggers: + - owm_math_v2 # extract math tags from html documents + - owm_latex_v2 # extract latex tags from html documents + - science_kw_v2 # extract the science keywords from the documents + skip: true # if none of the three taggers above returns any tags, skip the document + +post: + taggers: + - owmV2_FTsciV1_comb_vhth_qt # this is the tag that either looks for math/latex, or runs the fasttext science model (quantized) + - ft_dolma_doc_eng # language filter + skip: true # if the tagger above does not return any tags, skip the document + mode: all # filter out documents that do not have any tags + +skip_checks: true # this skips checking if the paths in `documents` exist +skip_duplicate_urls: true # this skips checking if the URLs in the documents are unique +batch_size: 10 # each process does 100 files at a time + +store: + html: false # do not keep the html in metadata + +work_dir: + input: ${oc.env:HOME}/progress-science-v1/${oc.env:SNAPSHOT_ID}/input + output: ${oc.env:HOME}/progress-science-v1/${oc.env:SNAPSHOT_ID}/output diff --git a/configs/crawl/science-v3.yaml b/configs/crawl/science-v3.yaml new file mode 100644 index 00000000..75138bc0 --- /dev/null +++ b/configs/crawl/science-v3.yaml @@ -0,0 +1,34 @@ +documents: "${d.sed:${d.split:${d.file:${d.cache:https://data.commoncrawl.org/crawl-data/${oc.env:SNAPSHOT_ID}/warc.paths.gz}}},crawl-data@s3://commoncrawl/crawl-data@,@}" # download list of warc files from Common Crawl (d.cache), read the file (d.file), split the file into lines (d.split), and add the bucket prefix by replacing the string "crawl-data" with "s3://commoncrawl/crawl-data" (d.sed) +destination: + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/fasttext_v2_${oc.env:SNAPSHOT_ID} # save the documents in the S3 path; note that you have to provide SNAPSHOT_ID as an environment variable +processes: ${d.procs:} # use the number of processors available on the machine; in practice, I override this value in the command line by using 40 processes on a machine with 370GB of RAM because each process consumes around 9GB of RAM +source_name: science_${oc.env:SNAPSHOT_ID} +fast_linearizer: fast-p-less-space # this is the fast linearizer +linearizer: openwebmath # this is the openwebmath linearizer (final text) + +min_text_length: 150 # minimum text length to keep the document + +pre: # these run before HTML linearization + taggers: + - owm_math_v2 # extract math tags from html documents + - owm_latex_v2 # extract latex tags from html documents + - science_kw_v2 # extract the science keywords from the documents + skip: true # if none of the three taggers above returns any tags, skip the document + +post: + taggers: + - owmV2_FTsciV1_comb_vvhth_qt # this is the tag that either looks for math/latex, or runs the fasttext science model (quantized) + - ft_dolma_doc_eng # language filter + skip: true # if the tagger above does not return any tags, skip the document + mode: all # filter out documents that do not have any tags + +skip_checks: true # this skips checking if the paths in `documents` exist +skip_duplicate_urls: true # this skips checking if the URLs in the documents are unique +batch_size: 10 # each process does 100 files at a time + +store: + html: false # do not keep the html in metadata + +work_dir: + input: ${oc.env:HOME}/progress-science-v1/${oc.env:SNAPSHOT_ID}/input + output: ${oc.env:HOME}/progress-science-v1/${oc.env:SNAPSHOT_ID}/output diff --git a/configs/crawl/science-v4.yaml b/configs/crawl/science-v4.yaml new file mode 100644 index 00000000..fd2fe844 --- /dev/null +++ b/configs/crawl/science-v4.yaml @@ -0,0 +1,47 @@ +# documents: "${d.sed:${d.split:${d.file:${d.cache:https://data.commoncrawl.org/crawl-data/${oc.env:SNAPSHOT_ID}/warc.paths.gz}}},crawl-data@s3://commoncrawl/crawl-data@,@}" # download list of warc files from Common Crawl (d.cache), read the file (d.file), split the file into lines (d.split), and add the bucket prefix by replacing the string "crawl-data" with "s3://commoncrawl/crawl-data" (d.sed) +documents: "${d.sed:${d.split:${d.file:${d.cache:https://data.commoncrawl.org/crawl-data/${oc.env:SNAPSHOT_ID}/warc.paths.gz}}},crawl-data@s3://commoncrawl/crawl-data@,@}" # download list of warc files from Common Crawl (d.cache), read the file (d.file), split the file into lines (d.split), and add the bucket prefix by replacing the string "crawl-data" with "s3://commoncrawl/crawl-data" (d.sed) + +destination: + - s3://ai2-llm/pretraining-data/sources/cc-science/v1/documents/fasttext_v2_${oc.env:SNAPSHOT_ID} # save the documents in the S3 path; note that you have to provide SNAPSHOT_ID as an environment variable +processes: ${d.procs:} # use the number of processors available on the machine; in practice, I override this value in the command line by using 40 processes on a machine with 370GB of RAM because each process consumes around 9GB of RAM +source_name: science_${oc.env:SNAPSHOT_ID} +fast_linearizer: fast-p-less-space # this is the fast linearizer +linearizer: openwebmath # this is the openwebmath linearizer (final text) + +min_text_length: 150 # minimum text length to keep the document + +pre: # these run before HTML linearization + taggers: + - owm_math_v2 # extract math tags from html documents + - owm_latex_v2 # extract latex tags from html documents + - science_kw_v2 # extract the science keywords from the documents + skip: true # if none of the three taggers above returns any tags, skip the document + +post: + taggers: + - owmV2_FTsciV1_comb_vvhth_qt # this is the tag that either looks for math/latex, or runs the fasttext science model (quantized) + - ft_dolma_doc_eng # language filter + skip: true # if the tagger above does not return any tags, skip the document + mode: all # filter out documents that do not have any tags + +skip_checks: true # this skips checking if the paths in `documents` exist +skip_duplicate_urls: true # this skips checking if the URLs in the documents are unique +batch_size: 10 # each process does 100 files at a time + +store: + html: false # do not keep the html in metadata + +work_dir: + input: ${oc.env:HOME}/progress-science-v1/${oc.env:SNAPSHOT_ID}/input + output: ${oc.env:HOME}/progress-science-v1/${oc.env:SNAPSHOT_ID}/output + + + +# 1. documents & dest based on file format +# "s3://ai2-llm/pretraining-data/sources/cc-science/v1_dd_ngram_doc_le030/documents/CC-MAIN-2019-04/" +# "cc-science-1529.json.gz" +# 2. pre post linearizer & taggers +# 3. other configs +# 4. other changes besides the yaml file (already had the tagger with fasttest2 in science.py) + +# fst1 - fst2 - dedup (avoid low quality deduped with high quality) diff --git a/configs/crawl/science.yaml b/configs/crawl/science.yaml new file mode 100644 index 00000000..d03c2c7f --- /dev/null +++ b/configs/crawl/science.yaml @@ -0,0 +1,29 @@ +documents: "${d.sed:${d.split:${d.file:${d.cache:https://data.commoncrawl.org/crawl-data/${oc.env:SNAPSHOT_ID}/warc.paths.gz}}},crawl-data@s3://commoncrawl/crawl-data@,@}" # download list of warc files from Common Crawl (d.cache), read the file (d.file), split the file into lines (d.split), and add the bucket prefix by replacing the string "crawl-data" with "s3://commoncrawl/crawl-data" (d.sed) +destination: + - s3://ai2-llm/pretraining-data/sources/cc-science/v0/documents/${oc.env:SNAPSHOT_ID} # save the documents in the S3 path; note that you have to provide SNAPSHOT_ID as an environment variable +processes: ${d.procs:} # use the number of processors available on the machine; in practice, I override this value in the command line by using 40 processes on a machine with 370GB of RAM because each process consumes around 9GB of RAM +source_name: science_${oc.env:SNAPSHOT_ID} +linearizer: fast-p # this is the fast linearizer + +pre: # these run before HTML linearization + taggers: + - owm_math_v2 # extract math tags from html documents + - owm_latex_v2 # extract latex tags from html documents + - science_kw_v2 # extract the science keywords from the documents + skip: true # if none of the three taggers above returns any tags, skip the document + +post: + taggers: + - owmV2_FTsciV1_comb_lth_qt # this is the tag that either looks for math/latex, or runs the fasttext science model (quantized) + skip: true # if the tagger above does not return any tags, skip the document + +skip_checks: true # this skips checking if the paths in `documents` exist +skip_duplicate_urls: true # this skips checking if the URLs in the documents are unique +batch_size: 100 # each process does 100 files at a time + +store: + html: true # keep the html in metadata + +work_dir: + input: /tmp/science/${oc.env:SNAPSHOT_ID}/input + output: /tmp/science/${oc.env:SNAPSHOT_ID}/output diff --git a/configs/dolma-v1_7/qc/c4_ngram_dedupe_pd_mix.yaml b/configs/dolma-v1_7/qc/c4_ngram_dedupe_pd_mix.yaml new file mode 100644 index 00000000..1909ee85 --- /dev/null +++ b/configs/dolma-v1_7/qc/c4_ngram_dedupe_pd_mix.yaml @@ -0,0 +1,43 @@ +streams: + - name: c4 + documents: + - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/* + attributes: + - cc_multi_bin + - dedupe_para_ngrams_13_1 + - tokenizer_repetitions_v2r2 + - pii_regex_with_counts_v2 + output: + max_size_in_bytes: 2_147_483_648 + path: s3://ai2-llm/pretraining-data/sources/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/documents + min_text_length: 25 # matches wikipedia + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + # Remove repetitions + - >- + (.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + + # Quality filter + - .attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.01 + + # PII filter + - .attributes.pii_regex_with_counts_v2__pii_regex_with_counts_v2__doc_count[0][-1] > 5 + syntax: jq + span_replacement: + - span: "$.attributes.dedupe_para_ngrams_13_1" + min_score: 0.8 + replacement: '' + +work_dir: + input: "/tmp/c4_ngram_dedupe_pd_mix2-fix/input" + output: "/tmp/c4_ngram_dedupe_pd_mix2-fix/output" + +processes: 188 diff --git a/configs/dolma-v1_7/qc/cc_mix-cc_multi_bin.yaml b/configs/dolma-v1_7/qc/cc_mix-cc_multi_bin.yaml new file mode 100644 index 00000000..4ded3f5b --- /dev/null +++ b/configs/dolma-v1_7/qc/cc_mix-cc_multi_bin.yaml @@ -0,0 +1,53 @@ +streams: + - name: cc_en_head + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_docpara_le030_decontam/documents/cc_en_head/* + attributes: + - cc_multi_bin + output: + max_size_in_bytes: 3894967296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_head + # discard_fields: + # - attributes + # - metadata + filter: + include: + - (.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1]) > 0.01 + syntax: jq + + + - name: cc_en_middle + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_docpara_le030_decontam/documents/cc_en_middle/* + attributes: + - cc_multi_bin + output: + max_size_in_bytes: 3894967296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_middle + # discard_fields: + # - attributes + # - metadata + filter: + include: + - (.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1]) > 0.01 + syntax: jq + + + - name: cc_en_tail + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_docpara_le030_decontam/documents/cc_en_tail/* + attributes: + - cc_multi_bin + output: + max_size_in_bytes: 3894967296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_tail + # discard_fields: + # - attributes + # - metadata + filter: + include: + - (.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1]) > 0.01 + syntax: jq + + +processes: 190 diff --git a/configs/dolma-v1_7/qc/cc_multi_bin.yaml b/configs/dolma-v1_7/qc/cc_multi_bin.yaml new file mode 100644 index 00000000..f2ac7550 --- /dev/null +++ b/configs/dolma-v1_7/qc/cc_multi_bin.yaml @@ -0,0 +1,207 @@ + +model_path: ${oc.env:HOME}/fasttext_models/cc_multi_bin.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "ws" +num_processes: 128 + + +model: + learning_rate: 0.1 + epochs: 10 + word_vector_size: 300 + context_window_size: 10 + min_word_occurrences: 10 + # min_char_ngram_length: 5 + # max_char_ngram_length: 6 + number_of_buckets: 200_000 + loss_function: "softmax" + # hash_count: 1 + + +streams: + - name: wikipedia_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: wikiref_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: smallweb_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + label: "hq" + sample: + train: 1.0 + - name: smallweb_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + label: "hq" + sample: + dev: 1.0 + - name: smallweb_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: commoncrawl_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/more-cc/*.gz + label: "lq" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/more-cc/*.gz + label: "lq" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/more-cc/*.gz + label: "lq" + sample: + test: 1.0 + + - name: openhermes + documents: + - s3://ai2-llm/pretraining-data/sources/teknium_OpenHermes-2.5/v0/documents/*.gz + label: "hq" + sample: + train: 0.8 + dev: 0.1 + test: 0.1 + + - name: academic_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + label: "hq" + sample: + train: 1.0 + - name: academic_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + label: "hq" + sample: + dev: 1.0 + - name: academic_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: books_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + label: "hq" + sample: + train: 1.0 + - name: books_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + label: "hq" + sample: + dev: 1.0 + - name: books_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: fakenews_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + label: "lq" + sample: + train: 1.0 + - name: fakenews_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + label: "lq" + sample: + dev: 1.0 + - name: fakenews_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: adult_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + label: "lq" + sample: + train: 1.0 + - name: adult_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + label: "lq" + sample: + dev: 1.0 + - name: adult_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: open-web-math_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + label: "hq" + sample: + train: 1.0 + - name: open-web-math_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + label: "hq" + sample: + dev: 1.0 + - name: open-web-math_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + label: "hq" + sample: + test: 1.0 diff --git a/configs/dolma-v1_7/qc/cc_multi_bin_v2.yaml b/configs/dolma-v1_7/qc/cc_multi_bin_v2.yaml new file mode 100644 index 00000000..49e0f1b1 --- /dev/null +++ b/configs/dolma-v1_7/qc/cc_multi_bin_v2.yaml @@ -0,0 +1,306 @@ + +model_path: ${oc.env:HOME}/fasttext_models/cc_multi_bin_v3.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "ws" +num_processes: 128 + + +model: + learning_rate: 0.1 + epochs: 10 + word_vector_size: 300 + context_window_size: 10 + min_word_occurrences: 10 + # min_char_ngram_length: 5 + # max_char_ngram_length: 6 + number_of_buckets: 200_000 + loss_function: "softmax" + # hash_count: 1 + + +streams: + - name: wikipedia_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: wikiref_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: smallweb_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + label: "hq" + sample: + train: 1.0 + - name: smallweb_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + label: "hq" + sample: + dev: 1.0 + - name: smallweb_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: commoncrawl_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/more-cc/*.gz + label: "lq" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/more-cc/*.gz + label: "lq" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/more-cc/*.gz + label: "lq" + sample: + test: 1.0 + + - name: openhermes + documents: + - s3://ai2-llm/pretraining-data/sources/teknium_OpenHermes-2.5/v0/documents/*.gz + label: "hq" + sample: + train: 0.8 + dev: 0.1 + test: 0.1 + + - name: academic_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + label: "hq" + sample: + train: 1.0 + - name: academic_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + label: "hq" + sample: + dev: 1.0 + - name: academic_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: books_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + label: "hq" + sample: + train: 1.0 + - name: books_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + label: "hq" + sample: + dev: 1.0 + - name: books_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: fakenews_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + label: "lq" + sample: + train: 1.0 + - name: fakenews_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + label: "lq" + sample: + dev: 1.0 + - name: fakenews_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: adult_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + label: "lq" + sample: + train: 1.0 + - name: adult_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + label: "lq" + sample: + dev: 1.0 + - name: adult_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + label: "lq" + sample: + test: 1.0 + + - name: cnn_dailymail + documents: + - s3://ai2-llm/pretraining-data/sources/summarization/v1/documents/cnn_dailymail/train.jsonl.gz + label: "hq" + sample: + train: 0.5 + + - name: cnn_dailymail + documents: + - s3://ai2-llm/pretraining-data/sources/summarization/v1/documents/cnn_dailymail/validation.jsonl.gz + label: "hq" + sample: + dev: 0.5 + + - name: cnn_dailymail + documents: + - s3://ai2-llm/pretraining-data/sources/summarization/v1/documents/cnn_dailymail/test.jsonl.gz + label: "hq" + sample: + test: 0.5 + + - name: bbc_news + documents: + - s3://ai2-llm/pretraining-data/sources/summarization/v1/documents/bbc_news/train/* + label: "hq" + sample: + train: 0.5 + test: 0.05 + dev: 0.05 + + - name: multi_news + documents: + - s3://ai2-llm/pretraining-data/sources/summarization/v1/documents/multi_news/train.jsonl.gz + label: "hq" + sample: + train: 0.5 + + - name: multi_news + documents: + - s3://ai2-llm/pretraining-data/sources/summarization/v1/documents/multi_news/validation.jsonl.gz + label: "hq" + sample: + dev: 0.5 + + - name: multi_news + documents: + - s3://ai2-llm/pretraining-data/sources/summarization/v1/documents/multi_news/test.jsonl.gz + label: "hq" + sample: + test: 0.5 + + - name: xsum + documents: + - s3://ai2-llm/pretraining-data/sources/summarization/v1/documents/xsum/train.jsonl.gz + label: "hq" + sample: + train: 0.5 + + - name: xsum + documents: + - s3://ai2-llm/pretraining-data/sources/summarization/v1/documents/xsum/validation.jsonl.gz + label: "hq" + sample: + dev: 0.5 + + - name: xsum + documents: + - s3://ai2-llm/pretraining-data/sources/summarization/v1/documents/xsum/test.jsonl.gz + label: "hq" + sample: + test: 0.5 + + + # - name: frontpage-news + # documents: + # - s3://ai2-llm/pretraining-data/sources/AndyReas-frontpage-news/v2/documents/hq/*.gz + # label: "hq" + # sample: + # train: 0.5 + # dev: 0.025 + # test: 0.025 + + - name: recipes_nlg + documents: + - s3://ai2-llm/pretraining-data/sources/RecipeNLG/v1/documents/*.gz + label: "hq" + sample: + train: 0.35 + dev: 0.035 + test: 0.035 + + - name: orca_math + documents: + - s3://ai2-llm/pretraining-data/sources/microsoft-orca-math-word-problems-200k/v1/documents/*.gz + label: "hq" + sample: + train: 0.8 + dev: 0.1 + test: 0.1 + + - name: open-web-math_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + label: "hq" + sample: + train: 1.0 + - name: open-web-math_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + label: "hq" + sample: + dev: 1.0 + - name: open-web-math_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + label: "hq" + sample: + test: 1.0 diff --git a/configs/dolma-v1_7/qc/cc_w_wr_sw_s2_nsfw_fake_math_bin_ws_300_2Ms.yaml b/configs/dolma-v1_7/qc/cc_w_wr_sw_s2_nsfw_fake_math_bin_ws_300_2Ms.yaml new file mode 100644 index 00000000..69512ab8 --- /dev/null +++ b/configs/dolma-v1_7/qc/cc_w_wr_sw_s2_nsfw_fake_math_bin_ws_300_2Ms.yaml @@ -0,0 +1,215 @@ + +model_path: '${oc.env:HOME}/fasttext_models/cc_w_wr_sw_s2_nsfw_fake_math_bin_ws_300_2Ms.bin' + +cache_dir: '${oc.env:HOME}/fasttext_data' +word_tokenizer: "ws" +num_processes: 128 + +model: + learning_rate: 0.2 + epochs: 10 + word_vector_size: 300 + pretrained_vectors: '${oc.env:HOME}/fasttext_models/crawl-300d-2M-subword.vec' + autotune: + enabled: false + metric: f1:__label__hq + duration: 300 + model_size: "1G" + + +streams: + - name: wikipedia_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: wikiref_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: smallweb_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + label: "hq" + sample: + train: 1.0 + - name: smallweb_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + label: "hq" + sample: + dev: 1.0 + - name: smallweb_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: commoncrawl_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/cc/*.gz + label: "lq" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/cc/*.gz + label: "lq" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/cc/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: academic_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + label: "hq" + sample: + train: 1.0 + - name: academic_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + label: "hq" + sample: + dev: 1.0 + - name: academic_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: books_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + label: "hq" + sample: + train: 1.0 + - name: books_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + label: "hq" + sample: + dev: 1.0 + - name: books_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + label: "hq" + sample: + test: 1.0 + + - name: fakenews_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + label: "lq" + sample: + train: 1.0 + - name: fakenews_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + label: "lq" + sample: + dev: 1.0 + - name: fakenews_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: adult_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + label: "lq" + sample: + train: 1.0 + - name: adult_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + label: "lq" + sample: + dev: 1.0 + - name: adult_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + label: "lq" + sample: + test: 1.0 + + - name: open-web-math_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + label: "hq" + sample: + train: 1.0 + - name: open-web-math_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + label: "hq" + sample: + dev: 1.0 + - name: open-web-math_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + label: "hq" + sample: + test: 1.0 + + # - name: refinedweb_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/refinedweb/*.gz + # label: "hq" + # sample: + # train: 1.0 + # - name: refinedweb_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/refinedweb/*.gz + # label: "hq" + # sample: + # dev: 1.0 + # - name: refinedweb_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/refinedweb/*.gz + # label: "hq" + # sample: + # test: 1.0 diff --git a/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws.yaml b/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws.yaml new file mode 100644 index 00000000..9dacf60f --- /dev/null +++ b/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws.yaml @@ -0,0 +1,190 @@ + +model_path: ${oc.env:HOME}/fasttext_models/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "ws" +num_processes: 128 + + +model: + learning_rate: 0.2 + epochs: 10 + # autotune: true + loss_function: "softmax" + +streams: + - name: wikipedia_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: wikiref_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: smallweb_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + label: "hq" + sample: + train: 1.0 + - name: smallweb_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + label: "hq" + sample: + dev: 1.0 + - name: smallweb_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: commoncrawl_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/cc/*.gz + label: "lq" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/cc/*.gz + label: "lq" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/cc/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: academic_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + label: "hq" + sample: + train: 1.0 + - name: academic_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + label: "hq" + sample: + dev: 1.0 + - name: academic_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: books_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + label: "hq" + sample: + train: 1.0 + - name: books_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + label: "hq" + sample: + dev: 1.0 + - name: books_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + label: "hq" + sample: + test: 1.0 + + - name: fakenews_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + label: "lq" + sample: + train: 1.0 + - name: fakenews_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + label: "lq" + sample: + dev: 1.0 + - name: fakenews_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: adult_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + label: "lq" + sample: + train: 1.0 + - name: adult_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + label: "lq" + sample: + dev: 1.0 + - name: adult_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + label: "lq" + sample: + test: 1.0 + + - name: open-web-math_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + label: "hq" + sample: + train: 1.0 + - name: open-web-math_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + label: "hq" + sample: + dev: 1.0 + - name: open-web-math_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + label: "hq" + sample: + test: 1.0 diff --git a/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_c36_w3_o5.yaml b/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_c36_w3_o5.yaml new file mode 100644 index 00000000..6243064b --- /dev/null +++ b/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_c36_w3_o5.yaml @@ -0,0 +1,219 @@ + +model_path: ${oc.env:HOME}/fasttext_models/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_c36_w3_o5.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "ws" +num_processes: 128 + + +model: + learning_rate: 0.5 + epochs: 5 + min_char_ngram_length: 3 + max_char_ngram_length: 6 + max_word_ngram_length: 3 + min_word_occurrences: 5 + word_vector_size: 300 + pretrained_vectors: ${oc.env:HOME}/fasttext_models/crawl-300d-2M-subword.vec + autotune: + enabled: false + metric: f1:__label__hq + duration: 300 + model_size: "1G" + + +streams: + - name: wikipedia_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: wikiref_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: smallweb_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + label: "hq" + sample: + train: 1.0 + - name: smallweb_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + label: "hq" + sample: + dev: 1.0 + - name: smallweb_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: commoncrawl_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/cc/*.gz + label: "lq" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/cc/*.gz + label: "lq" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/cc/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: academic_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + label: "hq" + sample: + train: 1.0 + - name: academic_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + label: "hq" + sample: + dev: 1.0 + - name: academic_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: books_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + label: "hq" + sample: + train: 1.0 + - name: books_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + label: "hq" + sample: + dev: 1.0 + - name: books_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + label: "hq" + sample: + test: 1.0 + + - name: fakenews_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + label: "lq" + sample: + train: 1.0 + - name: fakenews_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + label: "lq" + sample: + dev: 1.0 + - name: fakenews_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: adult_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + label: "lq" + sample: + train: 1.0 + - name: adult_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + label: "lq" + sample: + dev: 1.0 + - name: adult_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + label: "lq" + sample: + test: 1.0 + + - name: open-web-math_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + label: "hq" + sample: + train: 1.0 + - name: open-web-math_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + label: "hq" + sample: + dev: 1.0 + - name: open-web-math_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + label: "hq" + sample: + test: 1.0 + + # - name: refinedweb_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/refinedweb/*.gz + # label: "hq" + # sample: + # train: 1.0 + # - name: refinedweb_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/refinedweb/*.gz + # label: "hq" + # sample: + # dev: 1.0 + # - name: refinedweb_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/refinedweb/*.gz + # label: "hq" + # sample: + # test: 1.0 diff --git a/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_para.yaml b/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_para.yaml new file mode 100644 index 00000000..e23ab91a --- /dev/null +++ b/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_para.yaml @@ -0,0 +1,191 @@ + +model_path: ${oc.env:HOME}/fasttext_models/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_para.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "para_ws" +num_processes: 128 + + +model: + learning_rate: 0.5 + epochs: 5 + # autotune: true + # loss_function: "softmax" + pretrained_vectors: ${oc.env:HOME}/fasttext_models/crawl-300d-2M-subword.vec + +streams: + - name: wikipedia_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: wikiref_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: smallweb_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + label: "hq" + sample: + train: 1.0 + - name: smallweb_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + label: "hq" + sample: + dev: 1.0 + - name: smallweb_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: commoncrawl_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/cc/*.gz + label: "lq" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/cc/*.gz + label: "lq" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/cc/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: academic_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + label: "hq" + sample: + train: 1.0 + - name: academic_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + label: "hq" + sample: + dev: 1.0 + - name: academic_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: books_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + label: "hq" + sample: + train: 1.0 + - name: books_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + label: "hq" + sample: + dev: 1.0 + - name: books_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + label: "hq" + sample: + test: 1.0 + + - name: fakenews_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + label: "lq" + sample: + train: 1.0 + - name: fakenews_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + label: "lq" + sample: + dev: 1.0 + - name: fakenews_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: adult_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + label: "lq" + sample: + train: 1.0 + - name: adult_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + label: "lq" + sample: + dev: 1.0 + - name: adult_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + label: "lq" + sample: + test: 1.0 + + - name: open-web-math_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + label: "hq" + sample: + train: 1.0 + - name: open-web-math_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + label: "hq" + sample: + dev: 1.0 + - name: open-web-math_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + label: "hq" + sample: + test: 1.0 diff --git a/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_para_quant500.yaml b/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_para_quant500.yaml new file mode 100644 index 00000000..7916cc03 --- /dev/null +++ b/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_para_quant500.yaml @@ -0,0 +1,190 @@ + +model_path: ${oc.env:HOME}/fasttext_models/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_para_quant_inf8_1M.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "para_ws" +num_processes: 128 + +model: + epochs: 1 + learning_rate: 0.1 + features_cutoff: 1_000_000 + subvector_size: 8 + model_path: ${oc.env:HOME}/fasttext_models/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_300_2m_subwords_para.bin + +streams: + - name: wikipedia_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: wikiref_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: smallweb_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + label: "hq" + sample: + train: 1.0 + - name: smallweb_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + label: "hq" + sample: + dev: 1.0 + - name: smallweb_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: commoncrawl_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/cc/*.gz + label: "lq" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/cc/*.gz + label: "lq" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/cc/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: academic_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + label: "hq" + sample: + train: 1.0 + - name: academic_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + label: "hq" + sample: + dev: 1.0 + - name: academic_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: books_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + label: "hq" + sample: + train: 1.0 + - name: books_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + label: "hq" + sample: + dev: 1.0 + - name: books_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + label: "hq" + sample: + test: 1.0 + + - name: fakenews_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + label: "lq" + sample: + train: 1.0 + - name: fakenews_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + label: "lq" + sample: + dev: 1.0 + - name: fakenews_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: adult_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + label: "lq" + sample: + train: 1.0 + - name: adult_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + label: "lq" + sample: + dev: 1.0 + - name: adult_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + label: "lq" + sample: + test: 1.0 + + - name: open-web-math_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + label: "hq" + sample: + train: 1.0 + - name: open-web-math_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + label: "hq" + sample: + dev: 1.0 + - name: open-web-math_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + label: "hq" + sample: + test: 1.0 diff --git a/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_quant.yaml b/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_quant.yaml new file mode 100644 index 00000000..257e14bd --- /dev/null +++ b/configs/dolma-v1_7/qc/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_quant.yaml @@ -0,0 +1,190 @@ + +model_path: ${oc.env:HOME}/fasttext_models/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws_quant_inf8_1M.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "ws" +num_processes: 128 + +model: + epochs: 1 + learning_rate: 0.1 + features_cutoff: 6_000_000 + subvector_size: 16 + model_path: ${oc.env:HOME}/fasttext_models/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math_binary_ws.bin + +streams: + - name: wikipedia_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: wikiref_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: smallweb_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + label: "hq" + sample: + train: 1.0 + - name: smallweb_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + label: "hq" + sample: + dev: 1.0 + - name: smallweb_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: commoncrawl_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/cc/*.gz + label: "lq" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/cc/*.gz + label: "lq" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/cc/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: academic_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + label: "hq" + sample: + train: 1.0 + - name: academic_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + label: "hq" + sample: + dev: 1.0 + - name: academic_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: books_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + label: "hq" + sample: + train: 1.0 + - name: books_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + label: "hq" + sample: + dev: 1.0 + - name: books_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + label: "hq" + sample: + test: 1.0 + + - name: fakenews_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + label: "lq" + sample: + train: 1.0 + - name: fakenews_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + label: "lq" + sample: + dev: 1.0 + - name: fakenews_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: adult_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + label: "lq" + sample: + train: 1.0 + - name: adult_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + label: "lq" + sample: + dev: 1.0 + - name: adult_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + label: "lq" + sample: + test: 1.0 + + - name: open-web-math_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + label: "hq" + sample: + train: 1.0 + - name: open-web-math_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + label: "hq" + sample: + dev: 1.0 + - name: open-web-math_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + label: "hq" + sample: + test: 1.0 diff --git a/configs/dolma-v1_7/qc/check_quality_output_mix.yaml b/configs/dolma-v1_7/qc/check_quality_output_mix.yaml new file mode 100644 index 00000000..858be8b5 --- /dev/null +++ b/configs/dolma-v1_7/qc/check_quality_output_mix.yaml @@ -0,0 +1,218 @@ +streams: + # - name: cc_en_head + # documents: + # - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_head/cc_en_head-000* + # attributes: + # - suchin_whose_quality + # - rw_hrms_bin + # - rw_hrms_bin_v2 + # - cc_multi_bin + # output: + # max_size_in_bytes: 3894967296 + # path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/qc_exp/documents + # # discard_fields: + # # - attributes + # # - metadata + # filter: + # include: + # - (.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1]) <= 0.1 + # syntax: jq + + + # - name: cc_en_middle + # documents: + # - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_middle/cc_en_middle-000* + # attributes: + # - suchin_whose_quality + # - rw_hrms_bin + # - rw_hrms_bin_v2 + # - cc_multi_bin + # output: + # max_size_in_bytes: 3894967296 + # path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/qc_exp/documents + # # discard_fields: + # # - attributes + # # - metadata + # filter: + # include: + # - (.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1]) <= 0.1 + # syntax: jq + + + # - name: cc_en_tail + # documents: + # - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_tail/cc_en_tail-000* + # attributes: + # - suchin_whose_quality + # - rw_hrms_bin + # - rw_hrms_bin_v2 + # - cc_multi_bin + # output: + # max_size_in_bytes: 3894967296 + # path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/qc_exp/documents + # # discard_fields: + # # - attributes + # # - metadata + # filter: + # include: + # - (.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1]) <= 0.1 + # syntax: jq + + # - name: falcon + # documents: + # - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/13.jsonl.gz + # - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/343.jsonl.gz + # - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/370.jsonl.gz + # - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/394.jsonl.gz + # - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/440.jsonl.gz + # attributes: + # - cc_multi_bin + # - paloma_paragraphs + # - paloma_documents + # output: + # max_size_in_bytes: 2894967296 + # path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/qc_exp/documents + # filter: + # include: + # - (.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1]) <= 0.01 + # exclude: + # # Contaminated Data from Paloma + # - ".attributes.paloma_documents_bff_duplicates != null" + # - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + # syntax: jq + + # - name: c4 + # documents: + # - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/part_0012.jsonl.gz + # - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/part_0047.jsonl.gz + # - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/part_0112.jsonl.gz + # - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/part_0131.jsonl.gz + # - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/part_0163.jsonl.gz + # - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/part_0444.jsonl.gz + # - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/part_0545.jsonl.gz + # attributes: + # - cc_multi_bin + # - dedupe_para_ngrams_13_1 + # - tokenizer_repetitions_v2r2 + # - pii_regex_with_counts_v2 + # output: + # max_size_in_bytes: 2894967296 + # path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/qc_exp/documents + # filter: + # include: + # - >- + # ((.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.01) and + # ((.attributes.dedupe_para_ngrams_13_1 | length == 0) or + # ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3))) + # exclude: + # # Remove repetitions + # - >- + # (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + # (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + # syntax: jq + + + # - name: falcon + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/qc_exp/documents/falcon* + # attributes: + # - cc_multi_bin_v2 + # output: + # max_size_in_bytes: 2894967296 + # path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/qc_exp_2/documents + # filter: + # include: + # - .attributes != null + # syntax: jq + + # - name: c4 + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/qc_exp/documents/c4* + # attributes: + # - cc_multi_bin_v2 + # output: + # max_size_in_bytes: 2894967296 + # path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/qc_exp_2/documents + # filter: + # include: + # - .attributes != null + # syntax: jq + + + # - name: tulu_flan + # documents: + # - s3://ai2-llm/pretraining-data/sources/tulu_flan/v1-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/documents/train/60M-shots_all-upweight_1-dialog_false-sep_newline-train-03* + # attributes: + # - cc_multi_bin + # - jigsaw_hatespeech_sentence_v2 + # - jigsaw_nsfw_sencence_v2 + # - pii_regex_with_counts_fast_v2 + # - tokenizer_repetitions_v2r2 + # - whitespace_tokenizer_v1 + # output: + # max_size_in_bytes: 2894967296 + # path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/smol/documents + # filter: + # include: + # # - >- + # # ((.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.01) + # # # Remove repetitions + # - >- + # (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + # (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + # syntax: jq + + # - name: megawika + # documents: + # - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/en/en-03* + # attributes: + # - cc_multi_bin + # - jigsaw_hatespeech_sentence_v2 + # - jigsaw_nsfw_sencence_v2 + # - pii_regex_with_counts_fast_v2 + # - tokenizer_repetitions_v2r2 + # - whitespace_tokenizer_v1 + # output: + # max_size_in_bytes: 2894967296 + # path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/smol/documents + # filter: + # include: + # # - >- + # # ((.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.01) + # # # Remove repetitions + # - >- + # (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + # (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + # # - >- + # # (.attributes.jigsaw_hatespeech_sentence_v2__jigsaw_hatespeech_sentence_v2____label__toxic| map(.[2]) | max)) > 0.5 + # # - >- + # # (.attributes.jigsaw_nsfw_sencence_v2__jigsaw_nsfw_sencence_v2____label__nsfw| map(.[2]) | max)) > 0.5 + # - (.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.01) + # syntax: jq + + - name: reddit + documents: + - s3://ai2-llm/pretraining-data/sources/reddit/v5-dedupe-pii-nsfw-toxic/documents/reddit-v2-dedupe-pii-nsfw-0013.json.gz + - s3://ai2-llm/pretraining-data/sources/reddit/v5-dedupe-pii-nsfw-toxic/documents/reddit-v2-dedupe-pii-nsfw-0028.json.gz + - s3://ai2-llm/pretraining-data/sources/reddit/v5-dedupe-pii-nsfw-toxic/documents/reddit-v2-dedupe-pii-nsfw-0184.json.gz + - s3://ai2-llm/pretraining-data/sources/reddit/v5-dedupe-pii-nsfw-toxic/documents/reddit-v2-dedupe-pii-nsfw-0211.json.gz + - s3://ai2-llm/pretraining-data/sources/reddit/v5-dedupe-pii-nsfw-toxic/documents/reddit-v2-dedupe-pii-nsfw-0248.json.gz + attributes: + - cc_multi_bin + - dedupe_para_ngrams_13_1 + - tokenizer_repetitions_v2r2 + output: + max_size_in_bytes: 2894967296 + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/smol/documents + filter: + include: + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + - >- + (.attributes.dedupe_para_ngrams_13_1 | length != 0) and + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) > 0.3) + - (.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.01) + syntax: jq + +processes: 190 diff --git a/configs/dolma-v1_7/qc/finetune_fasttext.yaml b/configs/dolma-v1_7/qc/finetune_fasttext.yaml new file mode 100644 index 00000000..65165e97 --- /dev/null +++ b/configs/dolma-v1_7/qc/finetune_fasttext.yaml @@ -0,0 +1,122 @@ + +model_path: ${oc.env:HOME}/fasttext_models/cc_wiki_wikiref_owt2_pes2o.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "ws_lower" +num_processes: 128 + + +model: + learning_rate: 0.2 + epochs: 10 + # autotune: true + loss_function: "softmax" + +streams: + - name: wikipedia_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/train/dolma_wiki/*.gz + label: "wikipedia" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/valid/dolma_wiki/*.gz + label: "wikipedia" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/test/dolma_wiki/*.gz + label: "wikipedia" + sample: + test: 1.0 + - name: wikiref_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/train/megawika/*.gz + label: "wikiref" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/valid/megawika/*.gz + label: "wikiref" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/test/megawika/*.gz + label: "wikiref" + sample: + test: 1.0 + - name: openwebtext2_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/train/openwebtext2/*.gz + label: "openwebtext2" + sample: + train: 1.0 + - name: openwebtext2_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/valid/openwebtext2/*.gz + label: "openwebtext2" + sample: + dev: 1.0 + - name: openwebtext2_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/test/openwebtext2/*.gz + label: "openwebtext2" + sample: + test: 1.0 + - name: commoncrawl_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/train/cc/*.gz + label: "commoncrawl" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/valid/cc/*.gz + label: "commoncrawl" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/test/cc/*.gz + label: "commoncrawl" + sample: + test: 1.0 + - name: academic_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/train/dolma_pes2o/*.gz + label: "academic" + sample: + train: 1.0 + - name: academic_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/valid/dolma_pes2o/*.gz + label: "academic" + sample: + dev: 1.0 + - name: academic_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/test/dolma_pes2o/*.gz + label: "academic" + sample: + test: 1.0 + - name: books_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/train/books/*.gz + label: "books" + sample: + train: 1.0 + - name: books_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/valid/books/*.gz + label: "books" + sample: + dev: 1.0 + - name: books_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v0/documents/test/books/*.gz + label: "books" + sample: + test: 1.0 diff --git a/configs/dolma-v1_7/qc/finetune_fasttext_v1.yaml b/configs/dolma-v1_7/qc/finetune_fasttext_v1.yaml new file mode 100644 index 00000000..48f0ed0f --- /dev/null +++ b/configs/dolma-v1_7/qc/finetune_fasttext_v1.yaml @@ -0,0 +1,190 @@ + +model_path: ${oc.env:HOME}/fasttext_models/cc_wiki_wikiref_sw_pes2o_adult_fakenews_math.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "ws_lower" +num_processes: 128 + + +model: + learning_rate: 0.2 + epochs: 10 + # autotune: true + loss_function: "softmax" + +streams: + - name: wikipedia_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + label: "wikipedia" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + label: "wikipedia" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + label: "wikipedia" + sample: + test: 1.0 + + + - name: wikiref_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + label: "wikiref" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + label: "wikiref" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + label: "wikiref" + sample: + test: 1.0 + + + - name: smallweb_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + label: "smallweb" + sample: + train: 1.0 + - name: smallweb_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + label: "smallweb" + sample: + dev: 1.0 + - name: smallweb_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + label: "smallweb" + sample: + test: 1.0 + + + - name: commoncrawl_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/cc/*.gz + label: "commoncrawl" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/cc/*.gz + label: "commoncrawl" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/cc/*.gz + label: "commoncrawl" + sample: + test: 1.0 + + + - name: academic_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + label: "academic" + sample: + train: 1.0 + - name: academic_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + label: "academic" + sample: + dev: 1.0 + - name: academic_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + label: "academic" + sample: + test: 1.0 + + + - name: books_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + label: "books" + sample: + train: 1.0 + - name: books_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + label: "books" + sample: + dev: 1.0 + - name: books_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + label: "books" + sample: + test: 1.0 + + - name: fakenews_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + label: "fakenews" + sample: + train: 1.0 + - name: fakenews_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + label: "fakenews" + sample: + dev: 1.0 + - name: fakenews_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + label: "fakenews" + sample: + test: 1.0 + + + - name: adult_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + label: "adult" + sample: + train: 1.0 + - name: adult_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + label: "adult" + sample: + dev: 1.0 + - name: adult_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + label: "adult" + sample: + test: 1.0 + + - name: open-web-math_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + label: "math" + sample: + train: 1.0 + - name: open-web-math_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + label: "math" + sample: + dev: 1.0 + - name: open-web-math_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + label: "math" + sample: + test: 1.0 diff --git a/configs/dolma-v1_7/qc/flan_rep_mix.yaml b/configs/dolma-v1_7/qc/flan_rep_mix.yaml new file mode 100644 index 00000000..b47d1eec --- /dev/null +++ b/configs/dolma-v1_7/qc/flan_rep_mix.yaml @@ -0,0 +1,33 @@ +streams: + - name: tulu_flan + documents: + - s3://ai2-llm/pretraining-data/sources/tulu_flan/v1-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/documents/train/* + attributes: + - tokenizer_repetitions_v2r2 + - whitespace_tokenizer_v1 + output: + max_size_in_bytes: 500_000_000 + path: s3://ai2-llm/pretraining-data/sources/tulu_flan/v12decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/documents/train/* + min_text_length: 25 # matches wikipedia + # discard_fields: + # - attributes + # - metadata + filter: + exclude: + # Remove repetitions + - >- + (.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + + # Too few tokens + - .attributes.whitespace_tokenizer_v1__whitespace_tokenizer_v1__length[0][-1] < 25 + + # Too many tokens + - .attributes.whitespace_tokenizer_v1__whitespace_tokenizer_v1__length[0][-1] > 5000 + syntax: jq + +work_dir: + input: "/tmp/tulu_flan_mix/input" + output: "/tmp/tulu_flan_mix/output" + +processes: 188 diff --git a/configs/dolma-v1_7/qc/floret_cc_multi_bin.yaml b/configs/dolma-v1_7/qc/floret_cc_multi_bin.yaml new file mode 100644 index 00000000..eb099939 --- /dev/null +++ b/configs/dolma-v1_7/qc/floret_cc_multi_bin.yaml @@ -0,0 +1,207 @@ + +model_path: ${oc.env:HOME}/floret_models/floret_cc_multi_bin.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "ws" +num_processes: 128 + + +model: + learning_rate: 0.1 + epochs: 10 + word_vector_size: 300 + context_window_size: 10 + min_word_occurrences: 10 + # min_char_ngram_length: 5 + # max_char_ngram_length: 6 + number_of_buckets: 200_000 + loss_function: "softmax" + hash_count: 1 + + +streams: + - name: wikipedia_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: wikiref_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: smallweb_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + label: "hq" + sample: + train: 1.0 + - name: smallweb_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + label: "hq" + sample: + dev: 1.0 + - name: smallweb_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: commoncrawl_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/more-cc/*.gz + label: "lq" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/more-cc/*.gz + label: "lq" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/more-cc/*.gz + label: "lq" + sample: + test: 1.0 + + - name: openhermes + documents: + - s3://ai2-llm/pretraining-data/sources/teknium_OpenHermes-2.5/v0/documents/*.gz + label: "hq" + sample: + train: 0.8 + dev: 0.1 + test: 0.1 + + - name: academic_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + label: "hq" + sample: + train: 1.0 + - name: academic_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + label: "hq" + sample: + dev: 1.0 + - name: academic_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: books_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + label: "hq" + sample: + train: 1.0 + - name: books_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + label: "hq" + sample: + dev: 1.0 + - name: books_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: fakenews_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + label: "lq" + sample: + train: 1.0 + - name: fakenews_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + label: "lq" + sample: + dev: 1.0 + - name: fakenews_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: adult_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + label: "lq" + sample: + train: 1.0 + - name: adult_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + label: "lq" + sample: + dev: 1.0 + - name: adult_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: open-web-math_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + label: "hq" + sample: + train: 1.0 + - name: open-web-math_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + label: "hq" + sample: + dev: 1.0 + - name: open-web-math_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + label: "hq" + sample: + test: 1.0 diff --git a/configs/dolma-v1_7/qc/floret_suchin_whose_quality.yaml b/configs/dolma-v1_7/qc/floret_suchin_whose_quality.yaml new file mode 100644 index 00000000..5c52cea0 --- /dev/null +++ b/configs/dolma-v1_7/qc/floret_suchin_whose_quality.yaml @@ -0,0 +1,68 @@ + +model_path: ${oc.env:HOME}/fasttext_models/floret_suchin_whose_quality_v2.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "ws" +num_processes: 128 + + +model: + learning_rate: 0.1 + epochs: 10 + word_vector_size: 300 + context_window_size: 10 + min_word_occurrences: 10 + # min_char_ngram_length: 5 + # max_char_ngram_length: 6 + number_of_buckets: 50_000 + loss_function: "softmax" + hash_count: 1 + # min_char_ngram_length: 3 + # max_char_ngram_length: 6 + # max_word_ngram_length: 3 + # min_word_occurrences: 3 + # word_vector_size: 300 + # pretrained_vectors: ${oc.env:HOME}/fasttext_models/crawl-300d-2M-subword.vec + # autotune: + # enabled: false + # metric: f1:__label__hq + # duration: 300 + # model_size: "1G" + + +streams: + - name: wikiwebooks_train + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/train/WikiWebBooks/wikiwebbooks-0000.jsonl.gz' + label: "$.metadata.label" + sample: + train: 1.0 + - name: random_cc_train + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/train/Random_CC/random_cc-0000.jsonl.gz' + label: "$.metadata.label" + sample: + train: 1.0 + - name: wikiwebooks_valid + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/valid/WikiWebBooks/wikiwebbooks-0000.jsonl.gz' + label: "$.metadata.label" + sample: + dev: 1.0 + - name: random_cc_valid + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/valid/Random_CC/random_cc-0000.jsonl.gz' + label: "$.metadata.label" + sample: + dev: 1.0 + - name: wikiwebooks_test + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/test/WikiWebBooks/wikiwebbooks-0000.jsonl.gz' + label: "$.metadata.label" + sample: + test: 1.0 + - name: random_cc_test + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/test/Random_CC/random_cc-0000.jsonl.gz' + label: "$.metadata.label" + sample: + test: 1.0 diff --git a/configs/dolma-v1_7/qc/quant_cc_w_wr_sw_s2_nsfw_fake_math_bin_ws_300_2Ms.yaml b/configs/dolma-v1_7/qc/quant_cc_w_wr_sw_s2_nsfw_fake_math_bin_ws_300_2Ms.yaml new file mode 100644 index 00000000..24aa34df --- /dev/null +++ b/configs/dolma-v1_7/qc/quant_cc_w_wr_sw_s2_nsfw_fake_math_bin_ws_300_2Ms.yaml @@ -0,0 +1,210 @@ +model_path: '${oc.env:HOME}/fasttext_models/q2_8m-cc_w_wr_sw_s2_nsfw_fake_math_bin_ws_300_2Ms.bin' + +cache_dir: '${oc.env:HOME}/fasttext_data' +word_tokenizer: "ws" +num_processes: 128 + +model: + learning_rate: 0.1 + epochs: 1 + features_cutoff: 8_388_608 + subvector_size: 2 + model_path: '${oc.env:HOME}/fasttext_models/cc_w_wr_sw_s2_nsfw_fake_math_bin_ws_300_2Ms.bin' + + +streams: + - name: wikipedia_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikipedia_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikipedia_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: wikiref_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + label: "hq" + sample: + train: 1.0 + - name: wikiref_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + label: "hq" + sample: + dev: 1.0 + - name: wikiref_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: smallweb_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + label: "hq" + sample: + train: 1.0 + - name: smallweb_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + label: "hq" + sample: + dev: 1.0 + - name: smallweb_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: commoncrawl_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/cc/*.gz + label: "lq" + sample: + train: 1.0 + - name: commoncrawl_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/cc/*.gz + label: "lq" + sample: + dev: 1.0 + - name: commoncrawl_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/cc/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: academic_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + label: "hq" + sample: + train: 1.0 + - name: academic_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + label: "hq" + sample: + dev: 1.0 + - name: academic_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + label: "hq" + sample: + test: 1.0 + + + - name: books_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + label: "hq" + sample: + train: 1.0 + - name: books_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + label: "hq" + sample: + dev: 1.0 + - name: books_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + label: "hq" + sample: + test: 1.0 + + - name: fakenews_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + label: "lq" + sample: + train: 1.0 + - name: fakenews_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + label: "lq" + sample: + dev: 1.0 + - name: fakenews_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + label: "lq" + sample: + test: 1.0 + + + - name: adult_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + label: "lq" + sample: + train: 1.0 + - name: adult_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + label: "lq" + sample: + dev: 1.0 + - name: adult_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + label: "lq" + sample: + test: 1.0 + + - name: open-web-math_train + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + label: "hq" + sample: + train: 1.0 + - name: open-web-math_dev + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + label: "hq" + sample: + dev: 1.0 + - name: open-web-math_test + documents: + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + label: "hq" + sample: + test: 1.0 + + # - name: refinedweb_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/refinedweb/*.gz + # label: "hq" + # sample: + # train: 1.0 + # - name: refinedweb_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/refinedweb/*.gz + # label: "hq" + # sample: + # dev: 1.0 + # - name: refinedweb_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/refinedweb/*.gz + # label: "hq" + # sample: + # test: 1.0 diff --git a/configs/dolma-v1_7/qc/rw_hrms_bin.yaml b/configs/dolma-v1_7/qc/rw_hrms_bin.yaml new file mode 100644 index 00000000..a52665a0 --- /dev/null +++ b/configs/dolma-v1_7/qc/rw_hrms_bin.yaml @@ -0,0 +1,247 @@ + +model_path: ${oc.env:HOME}/fasttext_models/rw_hrms_bin.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "ws" +num_processes: 128 + + +model: + learning_rate: 0.1 + epochs: 10 + # min_char_ngram_length: 3 + # max_char_ngram_length: 6 + # max_word_ngram_length: 3 + min_word_occurrences: 3 + word_vector_size: 300 + # pretrained_vectors: ${oc.env:HOME}/fasttext_models/crawl-300d-2M-subword.vec + # autotune: + # enabled: false + # metric: f1:__label__hq + # duration: 300 + # model_size: "1G" + + +streams: + # - name: wikipedia_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_wiki/*.gz + # label: "hq" + # sample: + # train: 1.0 + # - name: wikipedia_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_wiki/*.gz + # label: "hq" + # sample: + # dev: 1.0 + # - name: wikipedia_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_wiki/*.gz + # label: "hq" + # sample: + # test: 1.0 + + + # - name: wikiref_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/megawika/*.gz + # label: "hq" + # sample: + # train: 1.0 + # - name: wikiref_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/megawika/*.gz + # label: "hq" + # sample: + # dev: 1.0 + # - name: wikiref_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/megawika/*.gz + # label: "hq" + # sample: + # test: 1.0 + + + # - name: smallweb_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + # label: "hq" + # sample: + # train: 1.0 + # - name: smallweb_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + # label: "hq" + # sample: + # dev: 1.0 + # - name: smallweb_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + # label: "hq" + # sample: + # test: 1.0 + + + # - name: commoncrawl_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/cc/*.gz + # label: "lq" + # sample: + # train: 1.0 + # - name: commoncrawl_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/cc/*.gz + # label: "lq" + # sample: + # dev: 1.0 + # - name: commoncrawl_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/cc/*.gz + # label: "lq" + # sample: + # test: 1.0 + + + # - name: academic_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/dolma_pes2o/*.gz + # label: "hq" + # sample: + # train: 1.0 + # - name: academic_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/dolma_pes2o/*.gz + # label: "hq" + # sample: + # dev: 1.0 + # - name: academic_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/dolma_pes2o/*.gz + # label: "hq" + # sample: + # test: 1.0 + + + # - name: books_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/books/*.gz + # label: "hq" + # sample: + # train: 1.0 + # - name: books_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/books/*.gz + # label: "hq" + # sample: + # dev: 1.0 + # - name: books_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/books/*.gz + # label: "hq" + # sample: + # test: 1.0 + + # - name: fakenews_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/fakenews/*.gz + # label: "lq" + # sample: + # train: 1.0 + # - name: fakenews_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/fakenews/*.gz + # label: "lq" + # sample: + # dev: 1.0 + # - name: fakenews_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/fakenews/*.gz + # label: "lq" + # sample: + # test: 1.0 + + + # - name: adult_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/adult/*.gz + # label: "lq" + # sample: + # train: 1.0 + # - name: adult_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/adult/*.gz + # label: "lq" + # sample: + # dev: 1.0 + # - name: adult_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/adult/*.gz + # label: "lq" + # sample: + # test: 1.0 + + # - name: open-web-math_train + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/open-web-math/*.gz + # label: "hq" + # sample: + # train: 1.0 + # - name: open-web-math_dev + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/open-web-math/*.gz + # label: "hq" + # sample: + # dev: 1.0 + # - name: open-web-math_test + # documents: + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/open-web-math/*.gz + # label: "hq" + # sample: + # test: 1.0 + + - name: refinedweb_train + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/refinedweb/*.gz + label: "lq" + sample: + train: 1.0 + - name: refinedweb_dev + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/refinedweb/*.gz + label: "lq" + sample: + dev: 1.0 + - name: refinedweb_test + documents: + - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/refinedweb/*.gz + label: "lq" + sample: + test: 1.0 + + # - name: smallweb_train + # documents: + # - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/train/smallweb/*.gz + # label: "lq" + # sample: + # train: 1.0 + # - name: smallweb_dev + # documents: + # - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/valid/smallweb/*.gz + # label: "lq" + # sample: + # dev: 1.0 + # - name: smallweb_test + # documents: + # - s3://ai2-llm/pretraining-data/sources/whose_quality_corpus/v1/documents/test/smallweb/*.gz + # label: "lq" + # sample: + # test: 1.0 + + - name: openhermes + documents: + - s3://ai2-llm/pretraining-data/sources/teknium_OpenHermes-2.5/v0/documents/*.gz + label: "hq" + sample: + train: 0.8 + dev: 0.1 + test: 0.1 diff --git a/configs/dolma-v1_7/qc/subset_tte_blocklist.yaml b/configs/dolma-v1_7/qc/subset_tte_blocklist.yaml new file mode 100644 index 00000000..1f609ddf --- /dev/null +++ b/configs/dolma-v1_7/qc/subset_tte_blocklist.yaml @@ -0,0 +1,89 @@ +streams: + - name: adult + documents: &documents + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/olmo-mix/v1_6-300G-decon/documents/c4/*.gz + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/olmo-mix/v1_6-300G-decon/documents/cc_en_head/*.gz + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/olmo-mix/v1_6-300G-decon/documents/cc_en_middle/*.gz + - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/olmo-mix/v1_6-300G-decon/documents/cc_en_tail/*.gz + attributes: + - url_filtering + output: &output + max_size_in_bytes: 38949672960 + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/v1_6_300G_decon_cc_only_adult + # discard_fields: + # - attributes + # - metadata + filter: + include: + - "$.attributes[?(@['url_filtering__blocklist_hosts_porn_v1__url'] && !@['url_filtering__allowlist_wikidata_cleaned_v1__url'])]" + - "$.attributes[?(@['url_filtering__domain_blocklist_utp_v1__url'] && !@['url_filtering__allowlist_wikidata_cleaned_v1__url'])]" + + - name: wikidata + documents: *documents + attributes: + - url_filtering + - random_number_v1 + output: + <<: *output + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/v1_6_300G_decon_cc_only_wikidata + filter: + include: + - "$.attributes[?(@['url_filtering__allowlist_wikidata_cleaned_v1__url'] && @.['random_number_v1__random_number_v1__random'][0][2] < 0.1)]" + + - name: gambling + documents: *documents + attributes: &attributes + - url_filtering + output: + <<: *output + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/v1_6_300G_decon_cc_only_gambling + filter: + include: + - "$.attributes[?(@['url_filtering__blocklist_hosts_gambling_v1__url'] && !@['url_filtering__allowlist_wikidata_cleaned_v1__url'])]" + + - name: adware_malware + documents: *documents + attributes: *attributes + output: + <<: *output + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/v1_6_300G_decon_cc_only_adware_malware + filter: + include: + - "$.attributes[?(@['url_filtering__blocklist_hosts_adware_malware_v1__url'] && !@['url_filtering__allowlist_wikidata_cleaned_v1__url'])]" + + - name: fakenews + documents: *documents + attributes: *attributes + output: + <<: *output + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/v1_6_300G_decon_cc_only_fakenews + filter: + include: + - "$.attributes[?(@['url_filtering__blocklist_hosts_fakenews_v1__url'])]" + + - name: social + documents: *documents + attributes: *attributes + output: + <<: *output + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/v1_6_300G_decon_cc_only_social + filter: + include: + - "$.attributes[?(@['url_filtering__blocklist_hosts_social_v1__url'])]" + + - name: cloudflare + documents: *documents + attributes: + - url_filtering + - random_number_v1 + output: + <<: *output + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/v1_6_300G_decon_cc_only_cloudflare + filter: + include: + - "$@.attributes[?(@['url_filtering__cloudflare_rank_v1__url'] && @['url_filtering__cloudflare_rank_v1__url'][0] && @['url_filtering__cloudflare_rank_v1__url'][0][2] <= 4.0 && @['random_number_v1__random_number_v1__random'][0][2] < 0.3)]" + +work_dir: + input: "/tmp/blocklist_qc/input" + output: "/tmp/blocklist_qc/output" +processes: 188 diff --git a/configs/dolma-v1_7/qc/subset_tte_blocklist_og.yaml b/configs/dolma-v1_7/qc/subset_tte_blocklist_og.yaml new file mode 100644 index 00000000..0383e9dd --- /dev/null +++ b/configs/dolma-v1_7/qc/subset_tte_blocklist_og.yaml @@ -0,0 +1,76 @@ +streams: + - name: adult + documents: &documents + - s3://ai2-llm/pretraining-data/sources/common-crawl/abl-cc-v1-small-dedup/documents/*.gz + attributes: + - url_filtering + output: &output + max_size_in_bytes: 52_179_272_016 + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/cc_dedup_adult + # discard_fields: + # - attributes + # - metadata + filter: + include: + - "$.attributes[?(@['url_filtering__blocklist_hosts_porn_v1__url'] && !@['url_filtering__allowlist_wikidata_cleaned_v1__url'])]" + - "$.attributes[?(@['url_filtering__domain_blocklist_utp_v1__url'] && !@['url_filtering__allowlist_wikidata_cleaned_v1__url'])]" + + - name: wikidata + documents: *documents + attributes: + - url_filtering + output: + <<: *output + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/cc_dedup_wikidata + filter: + include: + - "$.attributes[?(@['url_filtering__allowlist_wikidata_cleaned_v1__url'] && @.url_filtering__random_number_v1__random[0][2] < 0.01)]" + + - name: cloudflare + documents: *documents + attributes: + - url_filtering + output: + <<: *output + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/cc_dedup_cloudflare/ + filter: + include: + - "$@.attributes[?(@.url_filtering__cloudflare_rank_v1__url && @.url_filtering__cloudflare_rank_v1__url[0] && @.url_filtering__cloudflare_rank_v1__url[0][2] <= 4.0 && @.url_filtering__random_number_v1__random[0][2] < 0.015)]" + + + - name: gambling + documents: *documents + attributes: &attributes + - url_filtering + output: + <<: *output + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/cc_dedup_gambling/ + filter: + include: + - "$.attributes[?(@['url_filtering__blocklist_hosts_gambling_v1__url'] && !@['url_filtering__allowlist_wikidata_cleaned_v1__url'])]" + + - name: adware_malware + documents: *documents + attributes: *attributes + output: + <<: *output + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/cc_dedup_maladware/ + filter: + include: + - "$.attributes[?(@['url_filtering__blocklist_hosts_adware_malware_v1__url'] && !@['url_filtering__allowlist_wikidata_cleaned_v1__url'])]" + + - name: fakenews + documents: *documents + attributes: *attributes + output: + <<: *output + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/all/cc_dedup_fakenews/ + filter: + include: + - "$.attributes[?(@['url_filtering__blocklist_hosts_fakenews_v1__url'])]" + + +work_dir: + input: "/tmp/blocklist_qc/input" + output: "/tmp/blocklist_qc/output" +processes: 188 diff --git a/configs/dolma-v1_7/qc/subsets_test_eval.yaml b/configs/dolma-v1_7/qc/subsets_test_eval.yaml new file mode 100644 index 00000000..920af624 --- /dev/null +++ b/configs/dolma-v1_7/qc/subsets_test_eval.yaml @@ -0,0 +1,176 @@ +streams: + + - name: books + documents: + - s3://ai2-llm/pretraining-data/sources/pile/v1/documents/test/books3/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/test/books + max_size_in_bytes: 5_604_566_529 # about 10 files + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] < 2)]" + + - name: books + documents: + - s3://ai2-llm/pretraining-data/sources/pile/v1/documents/val/books3/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/valid/books + max_size_in_bytes: 5_604_566_529 # about 10 files + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] < 2)]" + + + - name: openwebtext2 + documents: + - s3://ai2-llm/pretraining-data/sources/pile/v1/documents/test/openwebtext2/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/test/openwebtext2 + max_size_in_bytes: 5_075_431_452 # about 10 files + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] < 2)]" + + + - name: openwebtext2 + documents: + - s3://ai2-llm/pretraining-data/sources/pile/v1/documents/val/openwebtext2/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/valid/openwebtext2 + max_size_in_bytes: 5_075_431_452 # about 10 files + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] < 2)]" + + + + - name: megawika + documents: + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/en/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/test/megawika + max_size_in_bytes: 2_332_602_944 # about 10 files + discard_fields: + - metadata + # - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] > 0.20 && @.random_number_v1__random_number_v1__random[0][2] < 0.21)]" + + - name: megawika + documents: + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/en/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/valid/megawika + max_size_in_bytes: 2_332_602_944 # about 10 files + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] > 0.30 && @.random_number_v1__random_number_v1__random[0][2] < 0.31)]" + + + - name: dolma_wiki + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/wiki/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/valid/dolma_wiki + max_size_in_bytes: 2_332_602_944 # about 10 files + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] > 0.35 && @.random_number_v1__random_number_v1__random[0][2] < 0.37)]" + + - name: dolma_wiki + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/wiki/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/test/dolma_wiki + max_size_in_bytes: 2_332_602_944 # about 10 files + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] > 0.45 && @.random_number_v1__random_number_v1__random[0][2] < 0.47)]" + + - name: dolma_pes2o + documents: &dolma_pes2o + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2ag/split=valid/part_id=0/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2ag/split=valid/part_id=1/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2ag/split=valid/part_id=2/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2ag/split=valid/part_id=3/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2ag/split=valid/part_id=4/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2ag/split=valid/part_id=5/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2ag/split=valid/part_id=6/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2ag/split=valid/part_id=7/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2ag/split=valid/part_id=8/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2ag/split=valid/part_id=9/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2orc/split=valid/part_id=0/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2orc/split=valid/part_id=1/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2orc/split=valid/part_id=2/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2orc/split=valid/part_id=3/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2orc/split=valid/part_id=4/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2orc/split=valid/part_id=5/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2orc/split=valid/part_id=6/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2orc/split=valid/part_id=7/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2orc/split=valid/part_id=8/*.gz + - s3://ai2-llm/pretraining-data/sources/s2/v3-fos/documents/dataset=s2orc/split=valid/part_id=9/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/valid/dolma_pes2o + max_size_in_bytes: 10_619_262_881 # about 10 files + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] < 0.20)]" + + - name: dolma_pes2o + documents: *dolma_pes2o + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/test/dolma_pes2o + max_size_in_bytes: 10_619_262_881 # about 10 files + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] > 0.80)]" + +processes: 20 diff --git a/configs/dolma-v1_7/qc/subsets_train.yaml b/configs/dolma-v1_7/qc/subsets_train.yaml new file mode 100644 index 00000000..1a3dff96 --- /dev/null +++ b/configs/dolma-v1_7/qc/subsets_train.yaml @@ -0,0 +1,47 @@ +streams: + - name: cc + documents: + - s3://ai2-llm/pretraining-data/sources/common-crawl/v0-small/documents/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/train/cc + max_size_in_bytes: 2_000_000_000 + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] < 0.002)]" + + - name: cc + documents: + - s3://ai2-llm/pretraining-data/sources/common-crawl/v0-small/documents/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/test/cc + max_size_in_bytes: 30_000_000_000 + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] > 0.002 && @.random_number_v1__random_number_v1__random[0][2] < 0.0021)]" + + - name: cc + documents: + - s3://ai2-llm/pretraining-data/sources/common-crawl/v0-small/documents/*.gz + attributes: + - random_number_v1 + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/documents/valid/cc + max_size_in_bytes: 30_000_000_000 + discard_fields: + - metadata + - attributes + filter: + include: + - "$.attributes[?(@.random_number_v1__random_number_v1__random[0][2] > 0.003 && @.random_number_v1__random_number_v1__random[0][2] < 0.0031)]" + +processes: 188 diff --git a/configs/dolma-v1_7/qc/suchin_whose_quality.yaml b/configs/dolma-v1_7/qc/suchin_whose_quality.yaml new file mode 100644 index 00000000..cfcdcd0f --- /dev/null +++ b/configs/dolma-v1_7/qc/suchin_whose_quality.yaml @@ -0,0 +1,60 @@ + +model_path: ${oc.env:HOME}/fasttext_models/suchin_whose_quality_v2.bin +cache_dir: ${oc.env:HOME}/fasttext_data +word_tokenizer: "ws" +num_processes: 128 + + +model: + learning_rate: 0.1 + epochs: 20 + # min_char_ngram_length: 3 + # max_char_ngram_length: 6 + # max_word_ngram_length: 3 + # min_word_occurrences: 3 + # word_vector_size: 300 + # pretrained_vectors: ${oc.env:HOME}/fasttext_models/crawl-300d-2M-subword.vec + # autotune: + # enabled: false + # metric: f1:__label__hq + # duration: 300 + # model_size: "1G" + + +streams: + - name: wikiwebooks_train + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/train/WikiWebBooks/wikiwebbooks-0000.jsonl.gz' + label: "$.metadata.label" + sample: + train: 1.0 + - name: random_cc_train + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/train/Random_CC/random_cc-0000.jsonl.gz' + label: "$.metadata.label" + sample: + train: 1.0 + - name: wikiwebooks_valid + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/valid/WikiWebBooks/wikiwebbooks-0000.jsonl.gz' + label: "$.metadata.label" + sample: + dev: 1.0 + - name: random_cc_valid + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/valid/Random_CC/random_cc-0000.jsonl.gz' + label: "$.metadata.label" + sample: + dev: 1.0 + - name: wikiwebooks_test + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/test/WikiWebBooks/wikiwebbooks-0000.jsonl.gz' + label: "$.metadata.label" + sample: + test: 1.0 + - name: random_cc_test + documents: + - '${oc.env:HOME}/ai2-llm/pretraining-data/sources/whose_quality_corpus/v2/test/Random_CC/random_cc-0000.jsonl.gz' + label: "$.metadata.label" + sample: + test: 1.0 diff --git a/configs/dolma-v1_7/shuffle-tokenizer.yaml b/configs/dolma-v1_7/shuffle-tokenizer.yaml new file mode 100644 index 00000000..d7c3150c --- /dev/null +++ b/configs/dolma-v1_7/shuffle-tokenizer.yaml @@ -0,0 +1,46 @@ +destination: ${oc.env:HOME}/ai2-llm/preprocessed/olmo-mix/v1_7-all-shuffled/allenai/gpt-neox-olmo-dolma-v1_5 +documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_6-decontaminated/documents/books/*.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_6-decontaminated/documents/pes2o/*.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_6-decontaminated/documents/wiki/*.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_6-decontaminated/documents/wiki/*.gz + - s3://ai2-llm/pretraining-data/sources/redpajama/v1_decon_fix/documents/train/arxiv/*.gz + - s3://ai2-llm/pretraining-data/sources/redpajama/v1_decon_fix/documents/train/stackexchange/*.gz + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/train/*.gz + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/train/*.gz + - s3://ai2-llm/pretraining-data/sources/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/documents/train/*.gz + - s3://ai2-llm/pretraining-data/sources/cc-news/v3/documents/*/*.gz + - s3://ai2-llm/pretraining-data/sources/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/documents/*.gz + - s3://ai2-llm/pretraining-data/sources/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/documents/*.gz + - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/documents/*gz + - s3://ai2-llm/pretraining-data/sources/megawika/v1/documents/*.gz + - s3://ai2-llm/pretraining-data/sources/starcoder/v0_decontaminated_doc_only/documents/*gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_head/cc_en_head-*0.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_head/cc_en_head-*2.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_head/cc_en_head-*4.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_head/cc_en_head-*6.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_head/cc_en_head-*8.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_middle/cc_en_middle-*0.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_middle/cc_en_middle-*2.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_middle/cc_en_middle-*4.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_middle/cc_en_middle-*6.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_middle/cc_en_middle-*8.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_tail/cc_en_tail-*0.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_tail/cc_en_tail-*2.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_tail/cc_en_tail-*4.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_tail/cc_en_tail-*6.json.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/documents/cc_en_tail/cc_en_tail-*8.json.gz + + +processes: 188 +seed: 3920 +max_size: 21_474_836_480 +ring_size: 8 +batch_size: 20_000 + +tokenizer: + name_or_path: allenai/gpt-neox-olmo-dolma-v1_5 + bos_token_id: null + eos_token_id: 50279 + pad_token_id: 1 + segment_before_tokenization: false diff --git a/configs/dolma-v1_7/sources/c4.sh b/configs/dolma-v1_7/sources/c4.sh new file mode 100644 index 00000000..ee8b0959 --- /dev/null +++ b/configs/dolma-v1_7/sources/c4.sh @@ -0,0 +1,33 @@ +set -ex + +NUMBER_OF_CORES=188 + +dolma tokens \ + --documents '${oc.env:HOME}/ai2-llm/pretraining-data/sources/c4/v1_dd_ngram_doc_le030/documents/*.gz' \ + --destination '${oc.env:HOME}/ai2-llm/preprocessed/c4/v1_dd_ngram_doc_le030/gpt-neox-olmo-dolma-v1_5' \ + --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ + --max_size '4_294_967_296' \ + --seed 0 \ + --tokenizer.eos_token_id 50279 \ + --tokenizer.pad_token_id 1 \ + --processes "${NUMBER_OF_CORES}" + +dolma tokens \ + --documents '${oc.env:HOME}/ai2-llm/pretraining-data/sources/c4/v1_dd_ngram_docpara_le030/documents/*.gz' \ + --destination '${oc.env:HOME}/ai2-llm/preprocessed/c4/v1_dd_ngram_docpara_le030/gpt-neox-olmo-dolma-v1_5' \ + --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ + --max_size '4_294_967_296' \ + --seed 0 \ + --tokenizer.eos_token_id 50279 \ + --tokenizer.pad_token_id 1 \ + --processes "${NUMBER_OF_CORES}" + +# dolma tokens \ +# --documents 's3://ai2-llm/pretraining-data/sources/olmo-mix/v1_6-decontaminated/documents/cc_en_tail/*.gz' \ +# --destination 's3://ai2-llm/preprocessed/olmo-mix/v1_6-decontaminated/cc_en_tail/gpt-neox-olmo-dolma-v1_5' \ +# --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ +# --max_size '20_147_483_648' \ +# --seed 0 \ +# --tokenizer.eos_token_id 50279 \ +# --tokenizer.pad_token_id 1 \ +# --processes "${NUMBER_OF_CORES}" diff --git a/configs/dolma-v1_7/sources/c4_dedupe/c4-mix.yaml b/configs/dolma-v1_7/sources/c4_dedupe/c4-mix.yaml new file mode 100644 index 00000000..7b088705 --- /dev/null +++ b/configs/dolma-v1_7/sources/c4_dedupe/c4-mix.yaml @@ -0,0 +1,23 @@ +streams: + - name: c4 + documents: + - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/part_000* + attributes: + - dedupe_para_ngrams_13_1 + output: &output + max_size_in_bytes: 38949672960 + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/c4/v1_dedupe/documents/10 + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length > 0) and + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) > 0.1) + syntax: jq + +work_dir: + input: "/tmp/c4_mix/input" + output: "/tmp/c4_mix/output" +processes: 188 diff --git a/configs/dolma-v1_7/sources/c4_dedupe/c4-ngrams-dedupe.yaml b/configs/dolma-v1_7/sources/c4_dedupe/c4-ngrams-dedupe.yaml new file mode 100644 index 00000000..0172230f --- /dev/null +++ b/configs/dolma-v1_7/sources/c4_dedupe/c4-ngrams-dedupe.yaml @@ -0,0 +1,25 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/*.gz + +dedupe: + name: dedupe_paragraph_ngrams + paragraphs: + attribute_name: dedupe_paragraph_ngrams + by_ngram: + ngram_length: 13 + stride: 6 + overlap_threshold: 0.5 + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/c4_bloom_filter.bin + read_only: false + # estimated doc count is obtained by counting number of words in paragraphs + # then dividing by 13 (ngram_length) and multiplying by 2 (for each ngram) + estimated_doc_count: 24_077_793_475 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/c4-ngrams-dedupe/input + output: /tmp/c4-ngrams-dedupe/output diff --git a/configs/dolma-v1_7/sources/c4_dedupe/c4-para-dedupe.yaml b/configs/dolma-v1_7/sources/c4_dedupe/c4-para-dedupe.yaml new file mode 100644 index 00000000..50676dfe --- /dev/null +++ b/configs/dolma-v1_7/sources/c4_dedupe/c4-para-dedupe.yaml @@ -0,0 +1,20 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/*.gz + +dedupe: + name: dedupe_paragraph + paragraphs: + attribute_name: dedupe_paragraph + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/c4_bloom_filter.bin + read_only: false + # estimating roughly 4 billion paragraphs + estimated_doc_count: 4_000_000_000 + desired_false_positive_rate: 1e-06 + +processes: 188 +work_dir: + input: /tmp/c4-para-dedupe/input + output: /tmp/c4-para-dedupe/output diff --git a/configs/dolma-v1_7/sources/c4_ngram_dedupe_mix.yaml b/configs/dolma-v1_7/sources/c4_ngram_dedupe_mix.yaml new file mode 100644 index 00000000..44129917 --- /dev/null +++ b/configs/dolma-v1_7/sources/c4_ngram_dedupe_mix.yaml @@ -0,0 +1,25 @@ +streams: + - name: c4 + documents: + - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/* + attributes: + - dedupe_para_ngrams_13_1 + output: + max_size_in_bytes: 1_073_741_824 + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/c4/v1_dd_ngram_doc_le030/documents + min_text_length: 25 # matches wikipedia + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + syntax: jq + +work_dir: + input: "/tmp/c4_ngram_dedupe_mix/input" + output: "/tmp/c4_ngram_dedupe_mix/output" + +processes: 188 diff --git a/configs/dolma-v1_7/sources/c4_ngram_dedupe_pd_mix.yaml b/configs/dolma-v1_7/sources/c4_ngram_dedupe_pd_mix.yaml new file mode 100644 index 00000000..c8354181 --- /dev/null +++ b/configs/dolma-v1_7/sources/c4_ngram_dedupe_pd_mix.yaml @@ -0,0 +1,29 @@ +streams: + - name: c4 + documents: + - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/* + attributes: + - dedupe_para_ngrams_13_1 + output: + max_size_in_bytes: 1_073_741_824 + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/c4/v1_dd_ngram_docpara_le030/documents + min_text_length: 25 # matches wikipedia + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + syntax: jq + span_replacement: + - span: "$.attributes.dedupe_para_ngrams_13_1" + min_score: 0.8 + replacement: '' + +work_dir: + input: "/tmp/c4_ngram_dedupe_pd_mix/input" + output: "/tmp/c4_ngram_dedupe_pd_mix/output" + +processes: 188 diff --git a/configs/dolma-v1_7/sources/cc_ngram_dedupe_mix.yaml b/configs/dolma-v1_7/sources/cc_ngram_dedupe_mix.yaml new file mode 100644 index 00000000..efcdc519 --- /dev/null +++ b/configs/dolma-v1_7/sources/cc_ngram_dedupe_mix.yaml @@ -0,0 +1,82 @@ +streams: + - name: cc_en_head + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_head/* + attributes: + - dedupe_para_ngrams_13_1 + - paloma_paragraphs + - paloma_documents + output: + max_size_in_bytes: 3894967296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_doc_le030_decontam/documents/cc_en_head + min_text_length: 25 # matches wikipedia + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + syntax: jq + + + - name: cc_en_middle + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_middle/* + attributes: + - dedupe_para_ngrams_13_1 + - paloma_paragraphs + - paloma_documents + output: + max_size_in_bytes: 3894967296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_doc_le030_decontam/documents/cc_en_middle + min_text_length: 25 # matches wikipedia + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + syntax: jq + + + - name: cc_en_tail + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_tail/* + attributes: + - dedupe_para_ngrams_13_1 + - paloma_paragraphs + - paloma_documents + output: + max_size_in_bytes: 3894967296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_doc_le030_decontam/documents/cc_en_tail + min_text_length: 25 # matches wikipedia + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + syntax: jq + + +work_dir: + input: "/tmp/cc_ngram_dedupe_mix/input" + output: "/tmp/cc_ngram_dedupe_mix/output" + + +processes: 190 diff --git a/configs/dolma-v1_7/sources/cc_ngram_dedupe_pd_mix.yaml b/configs/dolma-v1_7/sources/cc_ngram_dedupe_pd_mix.yaml new file mode 100644 index 00000000..c688af9a --- /dev/null +++ b/configs/dolma-v1_7/sources/cc_ngram_dedupe_pd_mix.yaml @@ -0,0 +1,93 @@ +streams: + - name: cc_en_head + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_head/* + attributes: + - dedupe_para_ngrams_13_1 + - paloma_paragraphs + - paloma_documents + output: + max_size_in_bytes: 3894967296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_docpara_le030_decontam/documents/cc_en_head + min_text_length: 25 # matches wikipedia + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + syntax: jq + span_replacement: + - span: "$.attributes.dedupe_para_ngrams_13_1" + min_score: 0.8 + replacement: '' + + + - name: cc_en_middle + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_middle/* + attributes: + - dedupe_para_ngrams_13_1 + - paloma_paragraphs + - paloma_documents + output: + max_size_in_bytes: 3894967296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_docpara_le030_decontam/documents/cc_en_middle + min_text_length: 25 # matches wikipedia + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + syntax: jq + span_replacement: + - span: "$.attributes.dedupe_para_ngrams_13_1" + min_score: 0.8 + replacement: '' + + + - name: cc_en_tail + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_tail/* + attributes: + - dedupe_para_ngrams_13_1 + - paloma_paragraphs + - paloma_documents + output: + max_size_in_bytes: 3894967296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_docpara_le030_decontam/documents/cc_en_tail + min_text_length: 25 # matches wikipedia + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + syntax: jq + span_replacement: + - span: "$.attributes.dedupe_para_ngrams_13_1" + min_score: 0.8 + replacement: '' + + +work_dir: + input: "/tmp/cc_ngram_dedupe_pd_mix/input" + output: "/tmp/cc_ngram_dedupe_pd_mix/output" + +processes: 190 diff --git a/configs/dolma-v1_7/sources/decontam/pp2_docs.yaml b/configs/dolma-v1_7/sources/decontam/pp2_docs.yaml new file mode 100644 index 00000000..c86a7c0a --- /dev/null +++ b/configs/dolma-v1_7/sources/decontam/pp2_docs.yaml @@ -0,0 +1,26 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/train/*.gz + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/test/*.gz + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/validation/*.gz + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/train/*.gz + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/test/*.gz + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/validation/*.gz + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/train/*.gz + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/test/*.gz + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/validation/*.gz + + +dedupe: + name: paloma_documents + documents: + attribute_name: paloma_documents_bff_duplicates + key: $.text + skip_empty: true + +bloom_filter: + read_only: true + estimated_doc_count: 188815 + desired_false_positive_rate: 1e-15 + file: ${oc.env:HOME}/perplexity/filters/paloma_documents.bin + +processes: 94 diff --git a/configs/dolma-v1_7/sources/decontam/pp2_para.yaml b/configs/dolma-v1_7/sources/decontam/pp2_para.yaml new file mode 100644 index 00000000..657e011e --- /dev/null +++ b/configs/dolma-v1_7/sources/decontam/pp2_para.yaml @@ -0,0 +1,15 @@ +destination: s3://ai2-llm/preprocessed/sources/open-web-math/v0/gpt-neox-olmo-dolma-v1_5 +documents: + - s3://ai2-llm/pretraining-data/sources/open-web-math/v0/documents/*.gz + # - ${oc.env:HOME}/ai2-llm/pretraining-data/sources/starcoder/v0/documents/*/*.json.gz + +processes: 32 +seed: 3920 +max_size: 2_147_483_648 + +tokenizer: + name_or_path: allenai/gpt-neox-olmo-dolma-v1_5 + bos_token_id: null + eos_token_id: 50279 + pad_token_id: 1 + segment_before_tokenization: false diff --git a/configs/dolma-v1_7/sources/falcon/falcon_13_1.yaml b/configs/dolma-v1_7/sources/falcon/falcon_13_1.yaml new file mode 100644 index 00000000..89cfdb0b --- /dev/null +++ b/configs/dolma-v1_7/sources/falcon/falcon_13_1.yaml @@ -0,0 +1,26 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/* + +dedupe: + name: dedupe_para_ngrams_13_1 + paragraphs: + attribute_name: dedupe_para_ngrams_13_1 + by_ngram: + ngram_length: 13 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: true + paragraph_separator: ${d.unescape:'\n\n'} + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/falcon_dedupe_para_ngrams_13_1.bin + read_only: false + # set to of words + estimated_doc_count: 561_403_831_170 + desired_false_positive_rate: 0.1 + +processes: 128 +work_dir: + input: /tmp/falcon_dedupe_para_ngrams_13_1/input + output: /tmp/falcon_dedupe_para_ngrams_13_1/output diff --git a/configs/dolma-v1_7/sources/falcon/falcon_13_1_sub.yaml b/configs/dolma-v1_7/sources/falcon/falcon_13_1_sub.yaml new file mode 100644 index 00000000..b01dd0fa --- /dev/null +++ b/configs/dolma-v1_7/sources/falcon/falcon_13_1_sub.yaml @@ -0,0 +1,26 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/* + +dedupe: + name: falcon_13_1_sub + paragraphs: + attribute_name: falcon_13_1_sub + by_ngram: + ngram_length: 13 + stride: 1 + overlap_threshold: 0.5 + skip_short_paragraphs: false + # paragraph_separator: ${d.unescape:'\n\n'} + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/falcon_dedupe_para_ngrams_13_1.bin + read_only: false + # set to of words + estimated_doc_count: 561_403_831_170 + desired_false_positive_rate: 0.1 + +processes: 128 +work_dir: + input: /tmp/falcon_dedupe_para_ngrams_13_1/input + output: /tmp/falcon_dedupe_para_ngrams_13_1/output diff --git a/configs/dolma-v1_7/sources/falcon/v1.yaml b/configs/dolma-v1_7/sources/falcon/v1.yaml new file mode 100644 index 00000000..2b1ff910 --- /dev/null +++ b/configs/dolma-v1_7/sources/falcon/v1.yaml @@ -0,0 +1,48 @@ +streams: + - name: falcon + documents: + - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/* + attributes: + - cc_multi_bin + # - dedupe_para_ngrams_13_1 + - paloma_paragraphs + - paloma_documents + # - pii_regex_with_counts_fast_v2 + # - tokenizer_repetitions_v2r2 + output: + max_size_in_bytes: 3_814_697_265 + path: s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v1-frac_005_100-qc_cc_multi_bin-paloma/documents + min_text_length: 25 # matches wikipedia + discard_fields: + - attributes + filter: + # include: + # - >- + # (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + # ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + # Contaminated Data from Paloma + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + + # # Remove repetitions + # - >- + # (.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + # (.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + + # Quality filter + - .attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.01 + + # # PII filter + # - .attributes.pii_regex_with_counts_v2__pii_regex_with_counts_v2__doc_count[0][-1] > 5 + syntax: jq + # span_replacement: + # - span: "$.attributes.dedupe_para_ngrams_13_1" + # min_score: 0.8 + # replacement: '' + +work_dir: + input: "/tmp/falcon_cc_multi_bin_mix_v2/input" + output: "/tmp/falcon_cc_multi_bin_mix_v2/output" + +processes: 188 diff --git a/configs/dolma-v1_7/sources/falcon/v2.yaml b/configs/dolma-v1_7/sources/falcon/v2.yaml new file mode 100644 index 00000000..e13c4f1b --- /dev/null +++ b/configs/dolma-v1_7/sources/falcon/v2.yaml @@ -0,0 +1,48 @@ +streams: + - name: falcon + documents: + - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/* + attributes: + - cc_multi_bin + - dedupe_para_ngrams_13_1 + - paloma_paragraphs + - paloma_documents + - pii_regex_with_counts_fast_v2 + - tokenizer_repetitions_v2r2 + output: + max_size_in_bytes: 3_814_697_265 + path: s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/documents + min_text_length: 25 # matches wikipedia + discard_fields: + - attributes + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + # Contaminated Data from Paloma + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + + # Remove repetitions + - >- + (.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + + # Quality filter + - .attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.01 + + # PII filter + - .attributes.pii_regex_with_counts_fast_v2__pii_regex_with_counts_fast_v2__doc_count[0][-1] > 5 + syntax: jq + span_replacement: + - span: "$.attributes.dedupe_para_ngrams_13_1" + min_score: 0.8 + replacement: '' + +work_dir: + input: "/tmp/falcon_cc_multi_bin_mix_v2/input" + output: "/tmp/falcon_cc_multi_bin_mix_v2/output" + +processes: 188 diff --git a/configs/dolma-v1_7/sources/megawika/decontam_docs.yaml b/configs/dolma-v1_7/sources/megawika/decontam_docs.yaml new file mode 100644 index 00000000..115b8a39 --- /dev/null +++ b/configs/dolma-v1_7/sources/megawika/decontam_docs.yaml @@ -0,0 +1,52 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/af/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/az/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/cs/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/de/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/en/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/et/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fa/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/gu/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/id/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/it/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ja/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ka/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/kk/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/km/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ko/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/lv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ml/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mn/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/my/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ne/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pl/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ps/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pt/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ro/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ru/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/sv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ta/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/th/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ur/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/vi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/xh/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/zh/* + +dedupe: + name: paloma_documents + documents: + attribute_name: paloma_documents_bff_duplicates + key: $.text + skip_empty: true + +bloom_filter: + read_only: true + estimated_doc_count: 188815 + desired_false_positive_rate: 1e-15 + file: ${oc.env:HOME}/paloma_documents.bin + +processes: 94 diff --git a/configs/dolma-v1_7/sources/megawika/decontam_para.yaml b/configs/dolma-v1_7/sources/megawika/decontam_para.yaml new file mode 100644 index 00000000..aeaffd31 --- /dev/null +++ b/configs/dolma-v1_7/sources/megawika/decontam_para.yaml @@ -0,0 +1,54 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/af/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/az/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/cs/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/de/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/en/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/et/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fa/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/gu/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/id/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/it/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ja/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ka/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/kk/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/km/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ko/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/lv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ml/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mn/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/my/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ne/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pl/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ps/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pt/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ro/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ru/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/sv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ta/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/th/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ur/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/vi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/xh/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/zh/* + + +dedupe: + name: paloma_paragraphs + paragraphs: + attribute_name: paloma_paragraphs_bff_duplicates + skip_empty: true + +bloom_filter: + read_only: true + estimated_doc_count: 2336120 + # size_in_bytes: 104857600 # 100 MB; smaller causes too many FPs + desired_false_positive_rate: 1e-15 + # file: s3://ai2-llm/bloom-filters/perplexity-suite-v3_option2.bin + file: ${oc.env:HOME}/ paloma_paragraphs.bin + +processes: 94 diff --git a/configs/dolma-v1_7/sources/megawika/dedupe_ngrams.yaml b/configs/dolma-v1_7/sources/megawika/dedupe_ngrams.yaml new file mode 100644 index 00000000..2563b992 --- /dev/null +++ b/configs/dolma-v1_7/sources/megawika/dedupe_ngrams.yaml @@ -0,0 +1,73 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/af/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/az/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/cs/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/de/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/en/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/et/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fa/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/gu/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/id/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/it/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ja/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ka/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/kk/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/km/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ko/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/lv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ml/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mn/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/my/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ne/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pl/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ps/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pt/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ro/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ru/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/sv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ta/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/th/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ur/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/vi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/xh/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/zh/* + +# dedupe: +# name: dedupe_para_ngrams_13_1 +# paragraphs: +# attribute_name: dedupe_para_ngrams_13_1 +# by_ngram: +# ngram_length: 13 +# stride: 1 +# overlap_threshold: 0.5 +# skip_short_paragraphs: false +# paragraph_separator: ${d.unescape:'\n\n'} +# skip_empty: true + +dedupe: + name: dedupe_para_ngrams_13_1_v2 + paragraphs: + attribute_name: dedupe_para_ngrams_13_1_v2 + by_ngram: + ngram_length: 13 + stride: 1 + overlap_threshold: 0.5 + # skip_short_paragraphs: false + # paragraph_separator: ${d.unescape:'\n\n'} + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/mega_wika_dedupe_para_ngrams_13_1_v2.bin + read_only: false + # set to of words + estimated_doc_count: 123_589_835_382 + desired_false_positive_rate: 0.01 + +processes: 128 +work_dir: + input: /tmp/mega_wika_dedupe_para_ngrams_13_1_v2/input + output: /tmp/mega_wika_dedupe_para_ngrams_13_1_v2/output diff --git a/configs/dolma-v1_7/sources/megawika/dedupe_url.yaml b/configs/dolma-v1_7/sources/megawika/dedupe_url.yaml new file mode 100644 index 00000000..7778e377 --- /dev/null +++ b/configs/dolma-v1_7/sources/megawika/dedupe_url.yaml @@ -0,0 +1,56 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/af/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/az/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/cs/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/de/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/en/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/et/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fa/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/gu/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/id/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/it/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ja/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ka/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/kk/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/km/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ko/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/lv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ml/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mn/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/my/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ne/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pl/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ps/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pt/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ro/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ru/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/sv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ta/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/th/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ur/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/vi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/xh/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/zh/* + +dedupe: + name: dedupe_by_url + documents: + attribute_name: dedupe_by_url + key: $.metadata.url + skip_empty: false + +bloom_filter: + file: ${oc.env:HOME}/url + read_only: false + # set to of words + estimated_doc_count: 100_000_000 + desired_false_positive_rate: 1e-4 + +processes: 128 +work_dir: + input: /tmp/mega_wika_dedupe_by_url/input + output: /tmp/mega_wika_dedupe_by_url/output diff --git a/configs/dolma-v1_7/sources/megawika/megawika_long.yaml b/configs/dolma-v1_7/sources/megawika/megawika_long.yaml new file mode 100644 index 00000000..6355013f --- /dev/null +++ b/configs/dolma-v1_7/sources/megawika/megawika_long.yaml @@ -0,0 +1,53 @@ +streams: + - name: megawika + documents: + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/af/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/az/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/cs/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/de/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/en/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/et/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fa/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/gu/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/id/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/it/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ja/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ka/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/kk/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/km/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ko/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/lv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ml/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mn/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/my/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ne/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pl/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ps/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pt/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ro/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ru/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/sv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ta/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/th/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ur/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/vi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/xh/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/zh/* + attributes: &attributes + - whitespace_tokenizer_v1 + output: &output + max_size_in_bytes: 500_000_000 + path: ${oc.env:HOME}/megawika_long + min_text_length: 25 # matches wikipedia + # discard_fields: + # - attributes + filter: &filter + include: + - .attributes.whitespace_tokenizer_v1__whitespace_tokenizer_v1__length[0][-1] > 100000 + syntax: jq + +processes: 188 diff --git a/configs/dolma-v1_7/sources/megawika/megawika_mix.yaml b/configs/dolma-v1_7/sources/megawika/megawika_mix.yaml new file mode 100644 index 00000000..2a60d1b7 --- /dev/null +++ b/configs/dolma-v1_7/sources/megawika/megawika_mix.yaml @@ -0,0 +1,176 @@ +streams: + - name: megawika + documents: + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/af/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/az/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/cs/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/de/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/en/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/et/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fa/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/fr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/gu/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/hr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/id/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/it/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ja/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ka/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/kk/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/km/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ko/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/lv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ml/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mn/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/mr/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/my/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ne/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pl/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ps/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/pt/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ro/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ru/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/sv/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ta/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/th/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/ur/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/vi/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/xh/* + - s3://ai2-llm/pretraining-data/sources/megawika/v0/documents/zh/* + attributes: &attributes + - dedupe_para_ngrams_13_1 + - dedupe_para_ngrams_13_1_v2 + - gopher_v1 + - c4_v2 + - whitespace_tokenizer_v1 + - paloma_paragraphs + - paloma_documents + - pii_regex_with_counts_fast_v2 + - ft_lang_id_en_doc_v2 + - cc_multi_bin + - dedupe_by_url + output: &output + max_size_in_bytes: 500_000_000 + path: s3://ai2-llm/pretraining-data/sources/megawika/v1/documents + min_text_length: 25 # matches wikipedia + discard_fields: + - attributes + filter: &filter + include: + # not a repeated doc + # - >- + # (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + # ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + - >- + (.attributes.dedupe_para_ngrams_13_1_v2 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1_v2 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + # Dedupe by URL + - ".attributes.dedupe_by_url != null" + + # Language Id + - .attributes.ft_lang_id_en_doc_v2__ft_lang_id_en_doc_v2__en[0][-1] < 0.5 + + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v1__gopher_v1__word_count != null) and + (.attributes.gopher_v1__gopher_v1__word_count[0][2] < 50) + - >- + (.attributes.gopher_v1__gopher_v1__word_count != null) and + (.attributes.gopher_v1__gopher_v1__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v1__gopher_v1__median_word_length != null) and + (.attributes.gopher_v1__gopher_v1__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v1__gopher_v1__median_word_length != null) and + (.attributes.gopher_v1__gopher_v1__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v1__gopher_v1__symbol_to_word_ratio != null) and + (.attributes.gopher_v1__gopher_v1__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v1__gopher_v1__required_word_count != null) and + (.attributes.gopher_v1__gopher_v1__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Contaminated Data from Paloma + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + + # Documents with too much PII + - .attributes.pii_regex_with_counts_fast_v2__pii_regex_with_counts_fast_v2__doc_count[0][2] > 5 + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + + # Quality filter + - .attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.1 + + # Too few tokens + - .attributes.whitespace_tokenizer_v1__whitespace_tokenizer_v1__length[0][-1] < 25 + syntax: jq + + span_replacement: + - span: "$.attributes.dedupe_para_ngrams_13_1" + min_score: 0.8 + replacement: '' + + +work_dir: + input: "/tmp/megawika/input" + output: "/tmp/megawika/output" + +processes: 188 diff --git a/configs/dolma-v1_7/sources/news/cc_news_mix.yaml b/configs/dolma-v1_7/sources/news/cc_news_mix.yaml new file mode 100644 index 00000000..d979f0cd --- /dev/null +++ b/configs/dolma-v1_7/sources/news/cc_news_mix.yaml @@ -0,0 +1,134 @@ +streams: + - name: cc_news + documents: + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_head/*.gz + attributes: &attributes + - dedupe_para_ngrams_13_1 + - gopher_v1 + - c4_v2 + - whitespace_tokenizer_v1 + - paloma_paragraphs + - paloma_documents + - blocklist_hosts_fakenews_v1 + - pii_regex_with_counts_fast_v2 + output: &output + max_size_in_bytes: 1_073_741_824 + path: s3://ai2-llm/pretraining-data/sources/cc-news/v2/documents/cc_en_head + discard_fields: + - attributes + filter: &filter + include: + # Dedupe filter + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + # At least 25 words + - .attributes.whitespace_tokenizer_v1__whitespace_tokenizer_v1__word_count[0][2] >= 25 + exclude: + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v1__gopher_v1__word_count != null) and + (.attributes.gopher_v1__gopher_v1__word_count[0][2] < 50) + - >- + (.attributes.gopher_v1__gopher_v1__word_count != null) and + (.attributes.gopher_v1__gopher_v1__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v1__gopher_v1__median_word_length != null) and + (.attributes.gopher_v1__gopher_v1__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v1__gopher_v1__median_word_length != null) and + (.attributes.gopher_v1__gopher_v1__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v1__gopher_v1__symbol_to_word_ratio != null) and + (.attributes.gopher_v1__gopher_v1__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v1__gopher_v1__required_word_count != null) and + (.attributes.gopher_v1__gopher_v1__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Contaminated Data from Paloma + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + + # Documents with too much PII + - .attributes.pii_regex_with_counts_fast_v2__pii_regex_with_counts_fast_v2__doc_count[0][2] > 5 + + # Remove Fake News websites + - .attributes.blocklist_hosts_fakenews_v1__blocklist_hosts_fakenews_v1__url != null + syntax: jq + + - name: cc_news + documents: + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_middle/*.gz + attributes: *attributes + output: + << : *output + path: s3://ai2-llm/pretraining-data/sources/cc-news/v2/documents/cc_en_middle + filter: *filter + + - name: cc_news + documents: + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_tail/*.gz + attributes: *attributes + output: + << : *output + path: s3://ai2-llm/pretraining-data/sources/cc-news/v2/documents/cc_en_tail + filter: *filter + +work_dir: + input: "/tmp/cc_news_v2/input" + output: "/tmp/cc_news_v2/output" + +processes: 188 diff --git a/configs/dolma-v1_7/sources/news/cc_news_mix_v3.yaml b/configs/dolma-v1_7/sources/news/cc_news_mix_v3.yaml new file mode 100644 index 00000000..917478a2 --- /dev/null +++ b/configs/dolma-v1_7/sources/news/cc_news_mix_v3.yaml @@ -0,0 +1,143 @@ +streams: + - name: cc_news + documents: + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_head/*.gz + attributes: &attributes + - dedupe_para_ngrams_13_1 + - gopher_v1 + - c4_v2 + - whitespace_tokenizer_v1 + - paloma_paragraphs + - paloma_documents + - blocklist_hosts_fakenews_v1 + - pii_regex_with_counts_fast_v2 + - cc_multi_bin + output: &output + max_size_in_bytes: 1_073_741_824 + path: s3://ai2-llm/pretraining-data/sources/cc-news/v3/documents/cc_en_head + discard_fields: + - attributes + filter: &filter + include: + # Dedupe filter + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + # At least 25 words + - .attributes.whitespace_tokenizer_v1__whitespace_tokenizer_v1__word_count[0][2] >= 25 + exclude: + # C4 Rules + - >- + (.attributes.c4_v2__c4_v2__has_curly_brace != null) and + (.attributes.c4_v2__c4_v2__has_curly_brace[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_lorem_ipsum != null) and + (.attributes.c4_v2__c4_v2__has_lorem_ipsum[0][2] > 0.5) + - >- + (.attributes.c4_v2__c4_v2__has_javascript != null) and + (.attributes.c4_v2__c4_v2__has_javascript[0][2] > 0.5) + + # Gopher Rules + - >- + (.attributes.gopher_v1__gopher_v1__word_count != null) and + (.attributes.gopher_v1__gopher_v1__word_count[0][2] < 50) + - >- + (.attributes.gopher_v1__gopher_v1__word_count != null) and + (.attributes.gopher_v1__gopher_v1__word_count[0][2] > 100000) + - >- + (.attributes.gopher_v1__gopher_v1__median_word_length != null) and + (.attributes.gopher_v1__gopher_v1__median_word_length[0][2] < 3) + - >- + (.attributes.gopher_v1__gopher_v1__median_word_length != null) and + (.attributes.gopher_v1__gopher_v1__median_word_length[0][2] > 10) + - >- + (.attributes.gopher_v1__gopher_v1__symbol_to_word_ratio != null) and + (.attributes.gopher_v1__gopher_v1__symbol_to_word_ratio[0][2] > 0.1) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_words_with_alpha_character != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_words_with_alpha_character[0][2] < 0.8) + - >- + (.attributes.gopher_v1__gopher_v1__required_word_count != null) and + (.attributes.gopher_v1__gopher_v1__required_word_count[0][2] < 2) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_starting_with_bullet_point != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_starting_with_bullet_point[0][2] > 0.9) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_ending_with_ellipsis != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_lines_ending_with_ellipsis[0][2] > 0.3) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_duplicate_lines != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_lines != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_lines[0][2] > 0.3) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_2gram != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_2gram[0][2] > 0.2) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_3gram != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_3gram[0][2] > 0.18) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_4gram != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_most_common_4gram[0][2] > 0.16) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_5grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_5grams[0][2] > 0.15) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_6grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_6grams[0][2] > 0.14) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_7grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_7grams[0][2] > 0.13) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_8grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_8grams[0][2] > 0.12) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_9grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_9grams[0][2] > 0.11) + - >- + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_10grams != null) and + (.attributes.gopher_v1__gopher_v1__fraction_of_characters_in_duplicate_10grams[0][2] > 0.10) + + # Contaminated Data from Paloma + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + + # Documents with too much PII + - .attributes.pii_regex_with_counts_fast_v2__pii_regex_with_counts_fast_v2__doc_count[0][2] > 5 + + # Remove Fake News websites + - .attributes.blocklist_hosts_fakenews_v1__blocklist_hosts_fakenews_v1__url != null + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition > 10) + + # Quality filter + - .attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.01 + syntax: jq + + - name: cc_news + documents: + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_middle/*.gz + attributes: *attributes + output: + << : *output + path: s3://ai2-llm/pretraining-data/sources/cc-news/v3/documents/cc_en_middle + filter: *filter + + - name: cc_news + documents: + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_tail/*.gz + attributes: *attributes + output: + << : *output + path: s3://ai2-llm/pretraining-data/sources/cc-news/v3/documents/cc_en_tail + filter: *filter + +work_dir: + input: "/tmp/cc_news_v2/input" + output: "/tmp/cc_news_v2/output" + +processes: 188 diff --git a/configs/dolma-v1_7/sources/news/decontam_docs.yaml b/configs/dolma-v1_7/sources/news/decontam_docs.yaml new file mode 100644 index 00000000..88bd2faa --- /dev/null +++ b/configs/dolma-v1_7/sources/news/decontam_docs.yaml @@ -0,0 +1,19 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_head/*.gz + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_middle/*.gz + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_tail/*.gz + +dedupe: + name: paloma_documents + documents: + attribute_name: paloma_documents_bff_duplicates + key: $.text + skip_empty: true + +bloom_filter: + read_only: true + estimated_doc_count: 188815 + desired_false_positive_rate: 1e-15 + file: ${oc.env:HOME}/perplexity/filters/paloma_documents.bin + +processes: 94 diff --git a/configs/dolma-v1_7/sources/news/decontam_para.yaml b/configs/dolma-v1_7/sources/news/decontam_para.yaml new file mode 100644 index 00000000..8cfb41ac --- /dev/null +++ b/configs/dolma-v1_7/sources/news/decontam_para.yaml @@ -0,0 +1,21 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_head/*.gz + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_middle/*.gz + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_tail/*.gz + + +dedupe: + name: paloma_paragraphs + paragraphs: + attribute_name: paloma_paragraphs_bff_duplicates + skip_empty: true + +bloom_filter: + read_only: true + estimated_doc_count: 2336120 + # size_in_bytes: 104857600 # 100 MB; smaller causes too many FPs + desired_false_positive_rate: 1e-15 + # file: s3://ai2-llm/bloom-filters/perplexity-suite-v3_option2.bin + file: ${oc.env:HOME}/perplexity/filters/paloma_paragraphs.bin + +processes: 94 diff --git a/configs/dolma-v1_7/sources/news/dedupe_ngrams.yaml b/configs/dolma-v1_7/sources/news/dedupe_ngrams.yaml new file mode 100644 index 00000000..66507f53 --- /dev/null +++ b/configs/dolma-v1_7/sources/news/dedupe_ngrams.yaml @@ -0,0 +1,26 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_head/*.gz + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_middle/*.gz + - s3://ai2-llm/pretraining-data/sources/cc-news/v1/documents/cc_en_tail/*.gz + +dedupe: + name: dedupe_para_ngrams_13_1 + paragraphs: + attribute_name: dedupe_para_ngrams_13_1 + by_ngram: + ngram_length: 13 + stride: 1 + overlap_threshold: 0.5 + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/cc_news_dedupe_para_ngrams_13_1.bin + read_only: false + # set to of words + estimated_doc_count: 14_472_049_720 + desired_false_positive_rate: 0.01 + +processes: 128 +work_dir: + input: /tmp/cc_news_dedupe_para_ngrams_13_1/input + output: /tmp/cc_news_dedupe_para_ngrams_13_1/output diff --git a/configs/dolma-v1_7/sources/ngram_dedupe/c4_13_1.yaml b/configs/dolma-v1_7/sources/ngram_dedupe/c4_13_1.yaml new file mode 100644 index 00000000..85d8577b --- /dev/null +++ b/configs/dolma-v1_7/sources/ngram_dedupe/c4_13_1.yaml @@ -0,0 +1,25 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/*.gz + +dedupe: + name: dedupe_para_ngrams_13_1 + paragraphs: + attribute_name: dedupe_para_ngrams_13_1 + by_ngram: + ngram_length: 13 + stride: 1 + overlap_threshold: 0.5 + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/c4_dedupe_para_ngrams_13_1.bin + read_only: false + # estimated doc count is obtained by counting number of words in paragraphs + # then dividing by 13 (ngram_length) and multiplying by 2 (for each ngram) + estimated_doc_count: 359_916_731_334 + desired_false_positive_rate: 0.1 + +processes: 188 +work_dir: + input: /tmp/c4_dedupe_para_ngrams_13_1/input + output: /tmp/c4_dedupe_para_ngrams_13_1/output diff --git a/configs/dolma-v1_7/sources/ngram_dedupe/cc_en_13_1.yaml b/configs/dolma-v1_7/sources/ngram_dedupe/cc_en_13_1.yaml new file mode 100644 index 00000000..52747ba3 --- /dev/null +++ b/configs/dolma-v1_7/sources/ngram_dedupe/cc_en_13_1.yaml @@ -0,0 +1,27 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_head/*.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_middle/*.gz + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_tail/*.gz + +dedupe: + name: dedupe_para_ngrams_13_1_full + paragraphs: + attribute_name: dedupe_para_ngrams_13_1_full + by_ngram: + ngram_length: 13 + stride: 1 + overlap_threshold: 0.5 + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/cc_en_para_ngrams_13_1.bin + read_only: false + # set to of words + estimated_doc_count: 2_089_187_516_474 + desired_false_positive_rate: 0.1 + size_in_bytes: 1_950_000_000_000 + +processes: 128 +work_dir: + input: /tmp/cc_en_head_dedupe_para_ngrams_13_1/input + output: /tmp/cc_en_head_dedupe_para_ngrams_13_1/output diff --git a/configs/dolma-v1_7/sources/ngram_dedupe/cc_head_13_1.yaml b/configs/dolma-v1_7/sources/ngram_dedupe/cc_head_13_1.yaml new file mode 100644 index 00000000..59175943 --- /dev/null +++ b/configs/dolma-v1_7/sources/ngram_dedupe/cc_head_13_1.yaml @@ -0,0 +1,24 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_head/*.gz + +dedupe: + name: dedupe_para_ngrams_13_1 + paragraphs: + attribute_name: dedupe_para_ngrams_13_1 + by_ngram: + ngram_length: 13 + stride: 1 + overlap_threshold: 0.5 + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/cc_en_head_dedupe_para_ngrams_13_1.bin + read_only: false + # set to of words + estimated_doc_count: 601_513_143_267 + desired_false_positive_rate: 0.1 + +processes: 128 +work_dir: + input: /tmp/cc_en_head_dedupe_para_ngrams_13_1/input + output: /tmp/cc_en_head_dedupe_para_ngrams_13_1/output diff --git a/configs/dolma-v1_7/sources/ngram_dedupe/cc_middle_13_1.yaml b/configs/dolma-v1_7/sources/ngram_dedupe/cc_middle_13_1.yaml new file mode 100644 index 00000000..21e3ca77 --- /dev/null +++ b/configs/dolma-v1_7/sources/ngram_dedupe/cc_middle_13_1.yaml @@ -0,0 +1,24 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_middle/*.gz + +dedupe: + name: dedupe_para_ngrams_13_1 + paragraphs: + attribute_name: dedupe_para_ngrams_13_1 + by_ngram: + ngram_length: 13 + stride: 1 + overlap_threshold: 0.5 + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/cc_en_middle_dedupe_para_ngrams_13_1.bin + read_only: false + # set to of words + estimated_doc_count: 596_000_959_655 + desired_false_positive_rate: 0.1 + +processes: 128 +work_dir: + input: /tmp/cc_en_middle_dedupe_para_ngrams_13_1/input + output: /tmp/cc_en_middle_dedupe_para_ngrams_13_1/output diff --git a/configs/dolma-v1_7/sources/ngram_dedupe/cc_tail_13_1.yaml b/configs/dolma-v1_7/sources/ngram_dedupe/cc_tail_13_1.yaml new file mode 100644 index 00000000..16ae623f --- /dev/null +++ b/configs/dolma-v1_7/sources/ngram_dedupe/cc_tail_13_1.yaml @@ -0,0 +1,24 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_tail/*.gz + +dedupe: + name: dedupe_para_ngrams_13_1 + paragraphs: + attribute_name: dedupe_para_ngrams_13_1 + by_ngram: + ngram_length: 13 + stride: 1 + overlap_threshold: 0.5 + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/cc_en_tail_dedupe_para_ngrams_13_1.bin + read_only: false + # set to of words + estimated_doc_count: 891_673_413_552 + desired_false_positive_rate: 0.1 + +processes: 128 +work_dir: + input: /tmp/cc_en_tail_dedupe_para_ngrams_13_1/input + output: /tmp/cc_en_tail_dedupe_para_ngrams_13_1/output diff --git a/configs/dolma-v1_7/sources/ngram_dedupe/reddit.yaml b/configs/dolma-v1_7/sources/ngram_dedupe/reddit.yaml new file mode 100644 index 00000000..69989833 --- /dev/null +++ b/configs/dolma-v1_7/sources/ngram_dedupe/reddit.yaml @@ -0,0 +1,24 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/reddit/v5-dedupe-pii-nsfw-toxic/documents/* + +dedupe: + name: dedupe_para_ngrams_13_1 + paragraphs: + attribute_name: dedupe_para_ngrams_13_1 + by_ngram: + ngram_length: 13 + stride: 1 + overlap_threshold: 0.5 + skip_empty: true + +bloom_filter: + file: ${oc.env:HOME}/reddit_dedupe_para_ngrams_13_1.bin + read_only: false + # set to of words + estimated_doc_count: 200_000_000_000 + desired_false_positive_rate: 0.1 + +processes: 128 +work_dir: + input: /tmp/reddit_dedupe_para_ngrams_13_1/input + output: /tmp/reddit_dedupe_para_ngrams_13_1/output diff --git a/configs/dolma-v1_7/sources/ngram_dedupe_sample.yaml b/configs/dolma-v1_7/sources/ngram_dedupe_sample.yaml new file mode 100644 index 00000000..2bdba8c7 --- /dev/null +++ b/configs/dolma-v1_7/sources/ngram_dedupe_sample.yaml @@ -0,0 +1,76 @@ +streams: + - name: cc_head_dd10 + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_head/cc_en_head-0091.json.gz + attributes: + - dedupe_para_ngrams_13_1 + output: + max_size_in_bytes: 38949672960 + path: ${oc.env:HOME}/duplicates + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length > 0) and + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) > 0.1) + syntax: jq + + + - name: cc_middle_dd10 + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_middle/cc_en_middle-0091.json.gz + attributes: + - dedupe_para_ngrams_13_1 + output: + max_size_in_bytes: 38949672960 + path: ${oc.env:HOME}/duplicates + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length > 0) and + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) > 0.1) + syntax: jq + + + - name: cc_tail_dd10 + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_5r2/documents/cc_en_tail/cc_en_tail-0091.json.gz + attributes: + - dedupe_para_ngrams_13_1 + output: + max_size_in_bytes: 38949672960 + path: ${oc.env:HOME}/duplicates + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length > 0) and + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) > 0.1) + syntax: jq + + - name: falcon_dd10 + documents: + - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/91* + attributes: + - dedupe_para_ngrams_13_1 + output: + max_size_in_bytes: 38949672960 + path: ${oc.env:HOME}/duplicates + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length > 0) and + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) > 0.1) + syntax: jq + +processes: 188 diff --git a/configs/dolma-v1_7/sources/olmo_mix.sh b/configs/dolma-v1_7/sources/olmo_mix.sh new file mode 100644 index 00000000..b2b5f760 --- /dev/null +++ b/configs/dolma-v1_7/sources/olmo_mix.sh @@ -0,0 +1,44 @@ +set -ex + + +export DOLMA_SUBSET=${1} + +# check if the subset is provided +if [ -z "${DOLMA_SUBSET}" ]; then + echo "Please provide the subset to process" + exit 1 +fi + +# reseerve 4 cores for the system +export NUMBER_OF_CORES=$(($(nproc) - 4)) + +dolma tokens \ + --documents 's3://ai2-llm/pretraining-data/sources/olmo-mix/${oc.env:DOLMA_SUBSET}/documents/cc_en_head/*.gz' \ + --destination '${oc.env:HOME}/ai2-llm/preprocessed/olmo-mix/${oc.env:DOLMA_SUBSET}/cc_en_head/gpt-neox-olmo-dolma-v1_5' \ + --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ + --max_size '20_147_483_648' \ + --seed 0 \ + --tokenizer.eos_token_id 50279 \ + --tokenizer.pad_token_id 1 \ + --processes "${NUMBER_OF_CORES}" + +dolma tokens \ + --documents 's3://ai2-llm/pretraining-data/sources/olmo-mix/${oc.env:DOLMA_SUBSET}/documents/cc_en_middle/*.gz' \ + --destination '${oc.env:HOME}/ai2-llm/preprocessed/olmo-mix/${oc.env:DOLMA_SUBSET}/cc_en_middle/gpt-neox-olmo-dolma-v1_5' \ + --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ + --max_size '20_147_483_648' \ + --seed 0 \ + --tokenizer.eos_token_id 50279 \ + --tokenizer.pad_token_id 1 \ + --processes "${NUMBER_OF_CORES}" + + +dolma tokens \ + --documents 's3://ai2-llm/pretraining-data/sources/olmo-mix/${oc.env:DOLMA_SUBSET}/documents/cc_en_tail/*.gz' \ + --destination '${oc.env:HOME}/ai2-llm/preprocessed/olmo-mix/${oc.env:DOLMA_SUBSET}/cc_en_tail/gpt-neox-olmo-dolma-v1_5' \ + --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ + --max_size '20_147_483_648' \ + --seed 0 \ + --tokenizer.eos_token_id 50279 \ + --tokenizer.pad_token_id 1 \ + --processes "${NUMBER_OF_CORES}" diff --git a/configs/dolma-v1_7/sources/proof-pile-2.sh b/configs/dolma-v1_7/sources/proof-pile-2.sh new file mode 100644 index 00000000..f34dea18 --- /dev/null +++ b/configs/dolma-v1_7/sources/proof-pile-2.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# get script directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + SOURCE="$(readlink "$SOURCE")" + # if $SOURCE was a relative symlink, we need to resolve it + # relative to the path where the symlink file was located + [[ $SOURCE != /* ]] && SOURCE="$SCRIPT_DIR/$SOURCE" +done +SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + + +set -ex + +NUMBER_OF_CORES=188 + +aws s3 cp s3://ai2-llm/bloom-filters/paloma_documents_20240219.bin "${HOME}/paloma_documents.bin" +aws s3 cp s3://ai2-llm/bloom-filters/paloma_paragraphs_20240219.bin "${HOME}/paloma_paragraphs.bin" + +dolma dedupe \ + --documents \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/train/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/validation/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/test/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/train/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/validation/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/test/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/train/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/validation/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/test/*.gz' \ + --dedupe.name 'paloma_documents' \ + --dedupe.documents.attribute_name 'paloma_documents' \ + --dedupe.documents.key '$.text' \ + --dedupe.skip_empty \ + --bloom_filter.file "${HOME}/paloma_documents.bin" \ + --bloom_filter.read_only \ + --bloom_filter.size_in_bytes 33557012 \ + --processes "${NUMBER_OF_CORES}" + +dolma dedupe \ + --documents \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/train/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/validation/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/test/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/train/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/validation/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/test/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/train/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/validation/*.gz' \ + 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/test/*.gz' \ + --dedupe.name 'paloma_paragraphs' \ + --dedupe.paragraphs.attribute_name 'paloma_paragraphs' \ + --dedupe.skip_empty \ + --bloom_filter.file "${HOME}/paloma_paragraphs.bin" \ + --bloom_filter.read_only \ + --bloom_filter.size_in_bytes 2099156 \ + --processes "${NUMBER_OF_CORES}" + +dolma -c ${SCRIPT_DIR}/proof-pile-2.yaml mix + +dolma tokens \ + --documents 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/algebraic-stack/train/*.gz' \ + --destination 's3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/gpt-neox-olmo-dolma-v1_5' \ + --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ + --max_size '2_147_483_648' \ + --seed 0 \ + --tokenizer.eos_token_id 50279 \ + --tokenizer.pad_token_id 1 \ + --processes "${NUMBER_OF_CORES}" + +dolma tokens \ + --documents 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/arxiv/train/*.gz' \ + --destination 's3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/gpt-neox-olmo-dolma-v1_5' \ + --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ + --max_size '2_147_483_648' \ + --seed 0 \ + --tokenizer.eos_token_id 50279 \ + --tokenizer.pad_token_id 1 \ + --processes "${NUMBER_OF_CORES}" + +dolma tokens \ + --documents 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/open-web-math/train/*.gz' \ + --destination 's3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5' \ + --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ + --max_size '2_147_483_648' \ + --seed 0 \ + --tokenizer.eos_token_id 50279 \ + --tokenizer.pad_token_id 1 \ + --processes "${NUMBER_OF_CORES}" diff --git a/configs/dolma-v1_7/sources/proof-pile-2.yaml b/configs/dolma-v1_7/sources/proof-pile-2.yaml new file mode 100644 index 00000000..26b674cc --- /dev/null +++ b/configs/dolma-v1_7/sources/proof-pile-2.yaml @@ -0,0 +1,95 @@ +streams: + + - name: algebraic-stack-train + documents: + - 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/train/*.gz' + attributes: &attributes + - paloma_paragraphs + - paloma_documents + output: &output + path: s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/algebraic-stack/train + max_size_in_bytes: 104_857_600 + filter: &filter_docs + exclude: + - "$@.attributes[?(@.paloma_documents && @.paloma_documents[0] && @.paloma_documents[0][2] >= 0)]" + + - name: algebraic-stack-validation + documents: + - 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/validation/*.gz' + attributes: *attributes + output: + <<: *output + path: s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/algebraic-stack/validation + filter: *filter_docs + + - name: algebraic-stack-test + documents: + - 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/algebraic-stack/test/*.gz' + attributes: *attributes + output: + <<: *output + path: s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/algebraic-stack/test + filter: *filter_docs + + + + + - name: arxiv-train + documents: + - 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/train/*.gz' + attributes: *attributes + output: + <<: *output + path: s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/arxiv/train + filter: &filter + exclude: + - "$@.attributes[?(@.paloma_paragraphs && @.paloma_paragraphs[0] && @.paloma_paragraphs[0][2] >= 0)]" + - "$@.attributes[?(@.paloma_documents && @.paloma_documents[0] && @.paloma_documents[0][2] >= 0)]" + + - name: arxiv-validation + documents: + - 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/validation/*.gz' + attributes: *attributes + output: + <<: *output + path: s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/arxiv/validation + filter: *filter + + - name: arxiv-test + documents: + - 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/arxiv/test/*.gz' + attributes: *attributes + output: + <<: *output + path: s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/arxiv/test + filter: *filter + + + + - name: open-web-math-train + documents: + - 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/train/*.gz' + attributes: *attributes + output: + <<: *output + path: s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/open-web-math/train + filter: *filter + + - name: open-web-math-validation + documents: + - 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/validation/*.gz' + attributes: *attributes + output: + <<: *output + path: s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/open-web-math/validation + filter: *filter + - name: open-web-math-test + documents: + - 's3://ai2-llm/pretraining-data/sources/proof-pile-2/v0/documents/open-web-math/test/*.gz' + attributes: *attributes + output: + <<: *output + path: s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/open-web-math/test + filter: *filter + +processes: 188 diff --git a/configs/dolma-v1_7/sources/reddit/decontam_docs.yaml b/configs/dolma-v1_7/sources/reddit/decontam_docs.yaml new file mode 100644 index 00000000..0b6495e1 --- /dev/null +++ b/configs/dolma-v1_7/sources/reddit/decontam_docs.yaml @@ -0,0 +1,17 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/reddit/v5-dedupe-pii-nsfw-toxic/documents/* + +dedupe: + name: paloma_documents + documents: + attribute_name: paloma_documents_bff_duplicates + key: $.text + skip_empty: true + +bloom_filter: + read_only: true + estimated_doc_count: 188815 + desired_false_positive_rate: 1e-15 + file: ${oc.env:HOME}/paloma_documents.bin + +processes: 94 diff --git a/configs/dolma-v1_7/sources/reddit/decontam_para.yaml b/configs/dolma-v1_7/sources/reddit/decontam_para.yaml new file mode 100644 index 00000000..d7fc521b --- /dev/null +++ b/configs/dolma-v1_7/sources/reddit/decontam_para.yaml @@ -0,0 +1,19 @@ +documents: + - s3://ai2-llm/pretraining-data/sources/reddit/v5-dedupe-pii-nsfw-toxic/documents/* + + +dedupe: + name: paloma_paragraphs + paragraphs: + attribute_name: paloma_paragraphs_bff_duplicates + skip_empty: true + +bloom_filter: + read_only: true + estimated_doc_count: 2336120 + # size_in_bytes: 104857600 # 100 MB; smaller causes too many FPs + desired_false_positive_rate: 1e-15 + # file: s3://ai2-llm/bloom-filters/perplexity-suite-v3_option2.bin + file: ${oc.env:HOME}/paloma_paragraphs.bin + +processes: 94 diff --git a/configs/dolma-v1_7/sources/reddit/reddit_mix_dedup.yaml b/configs/dolma-v1_7/sources/reddit/reddit_mix_dedup.yaml new file mode 100644 index 00000000..7502d5b0 --- /dev/null +++ b/configs/dolma-v1_7/sources/reddit/reddit_mix_dedup.yaml @@ -0,0 +1,30 @@ +streams: + - name: reddit + documents: + - s3://ai2-llm/pretraining-data/sources/reddit/v5-dedupe-pii-nsfw-toxic/documents/* + attributes: + - dedupe_para_ngrams_13_1 + - tokenizer_repetitions_v2r2 + - paloma_paragraphs + - paloma_documents + output: + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/documents + max_size_in_bytes: 4294967296 + min_text_length: 25 + discard_fields: + - attributes + filter: + include: + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition == null) or + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] <= 10) + - >- + (.attributes.dedupe_para_ngrams_13_1 | length != 0) and + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + # - (.attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.01) + exclude: + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + syntax: jq + +processes: 190 diff --git a/configs/dolma-v1_7/sources/redpajama-subsets.sh b/configs/dolma-v1_7/sources/redpajama-subsets.sh new file mode 100644 index 00000000..5cea1373 --- /dev/null +++ b/configs/dolma-v1_7/sources/redpajama-subsets.sh @@ -0,0 +1,26 @@ + +dolma tokens \ + --documents '${oc.env:HOME}/ai2-llm/pretraining-data/sources/redpajama/v1_decon_fix/documents/train/arxiv/*gz' \ + --destination '${oc.env:HOME}/ai2-llm/preprocessed/redpajama_v1_decon_fix/arxiv/gpt-neox-olmo-dolma-v1_5' \ + --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ + --max_size '2_147_483_648' \ + --seed 0 \ + --tokenizer.eos_token_id 50279 \ + --tokenizer.pad_token_id 1 \ + --processes 32 \ + --files_per_process 4 \ + --ring_size 4 + + + +dolma tokens \ + --documents '${oc.env:HOME}/ai2-llm/pretraining-data/sources/redpajama/v1_decon_fix/documents/train/stackexchange/*gz' \ + --destination '${oc.env:HOME}/ai2-llm/preprocessed/redpajama_v1_decon_fix/stackexchange/gpt-neox-olmo-dolma-v1_5' \ + --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ + --max_size '2_147_483_648' \ + --seed 0 \ + --tokenizer.eos_token_id 50279 \ + --tokenizer.pad_token_id 1 \ + --processes 32 \ + --files_per_process 4 \ + --ring_size 4 diff --git a/configs/dolma-v1_7/sources/redpajama-subsets.yaml b/configs/dolma-v1_7/sources/redpajama-subsets.yaml new file mode 100644 index 00000000..1662383c --- /dev/null +++ b/configs/dolma-v1_7/sources/redpajama-subsets.yaml @@ -0,0 +1,31 @@ +streams: + + - name: arxiv + documents: + - s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=arxiv/*.gz + attributes: + - paloma_paragraphs + - paloma_documents + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/redpajama/v1_decon_fix/documents/train/arxiv + max_size_in_bytes: 104_857_600 + filter: + exclude: + - "$@.attributes[?(@.paloma_paragraphs && @.paloma_paragraphs[0] && @.paloma_paragraphs[0][2] >= 0)]" + - "$@.attributes[?(@.paloma_documents && @.paloma_documents[0] && @.paloma_documents[0][2] >= 0)]" + + - name: stackexchange + documents: + - s3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=stackexchange/*.gz + attributes: + - paloma_paragraphs + - paloma_documents + output: + path: ${oc.env:HOME}/ai2-llm/pretraining-data/sources/redpajama/v1_decon_fix/documents/train/stackexchange + max_size_in_bytes: 26_214_400 + filter: + exclude: + - "$@.attributes[?(@.paloma_paragraphs && @.paloma_paragraphs[0] && @.paloma_paragraphs[0][2] >= 0)]" + - "$@.attributes[?(@.paloma_documents && @.paloma_documents[0] && @.paloma_documents[0][2] >= 0)]" + +processes: 96 diff --git a/configs/dolma-v1_7/sources/rp-stackexchange.sh b/configs/dolma-v1_7/sources/rp-stackexchange.sh new file mode 100644 index 00000000..fbd57099 --- /dev/null +++ b/configs/dolma-v1_7/sources/rp-stackexchange.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# get script directory +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + SOURCE="$(readlink "$SOURCE")" + # if $SOURCE was a relative symlink, we need to resolve it + # relative to the path where the symlink file was located + [[ $SOURCE != /* ]] && SOURCE="$SCRIPT_DIR/$SOURCE" +done +SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + + +set -ex + +NUMBER_OF_CORES=188 + +aws s3 cp s3://ai2-llm/bloom-filters/paloma_documents_20240219.bin "${HOME}/paloma_documents.bin" +aws s3 cp s3://ai2-llm/bloom-filters/paloma_paragraphs_20240219.bin "${HOME}/paloma_paragraphs.bin" + +dolma dedupe \ + --documents \ + 's3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=stackexchange/*.gz' \ + --dedupe.name 'paloma_documents' \ + --dedupe.documents.attribute_name 'paloma_documents' \ + --dedupe.documents.key '$.text' \ + --dedupe.skip_empty \ + --bloom_filter.file "${HOME}/paloma_documents.bin" \ + --bloom_filter.read_only \ + --bloom_filter.size_in_bytes 33557012 \ + --processes "${NUMBER_OF_CORES}" + +dolma dedupe \ + --documents \ + 's3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=stackexchange/*.gz' \ + --dedupe.name 'paloma_paragraphs' \ + --dedupe.paragraphs.attribute_name 'paloma_paragraphs' \ + --dedupe.skip_empty \ + --bloom_filter.file "${HOME}/paloma_paragraphs.bin" \ + --bloom_filter.read_only \ + --bloom_filter.size_in_bytes 2099156 \ + --processes "${NUMBER_OF_CORES}" + +dolma -c ${SCRIPT_DIR}/rp-stackexchange.yaml mix + +dolma tokens \ + --documents 's3://ai2-llm/pretraining-data/sources/redpajama/v1_decontaminated/documents/stackexchange*.gz' \ + --destination '${oc.env:HOME}/ai2-llm/preprocessed/redpajama_stackexchange_only/v1_decontaminated/gpt-neox-olmo-dolma-v1_5' \ + --tokenizer.name_or_path 'allenai/gpt-neox-olmo-dolma-v1_5' \ + --max_size '2_147_483_648' \ + --seed 0 \ + --tokenizer.eos_token_id 50279 \ + --tokenizer.pad_token_id 1 \ + --processes "${NUMBER_OF_CORES}" diff --git a/configs/dolma-v1_7/sources/rp-stackexchange.yaml b/configs/dolma-v1_7/sources/rp-stackexchange.yaml new file mode 100644 index 00000000..2adada44 --- /dev/null +++ b/configs/dolma-v1_7/sources/rp-stackexchange.yaml @@ -0,0 +1,17 @@ +streams: + + - name: stackexchange + documents: + - 's3://ai2-llm/pretraining-data/sources/redpajama/v1/documents/split=train/dataset=stackexchange/*.gz' + attributes: &attributes + - paloma_paragraphs + - paloma_documents + output: &output + path: s3://ai2-llm/pretraining-data/sources/redpajama/v1_decontaminated/documents + max_size_in_bytes: 104_857_600 + filter: &filter_docs + exclude: + - "$@.attributes[?(@.paloma_paragraphs && @.paloma_paragraphs[0] && @.paloma_paragraphs[0][2] >= 0)]" + - "$@.attributes[?(@.paloma_documents && @.paloma_documents[0] && @.paloma_documents[0][2] >= 0)]" + +processes: 188 diff --git a/configs/dolma-v1_7/sources/superhigh-25.yaml b/configs/dolma-v1_7/sources/superhigh-25.yaml new file mode 100644 index 00000000..62177a96 --- /dev/null +++ b/configs/dolma-v1_7/sources/superhigh-25.yaml @@ -0,0 +1,160 @@ +streams: + - name: falcon + documents: + - s3://ai2-llm/pretraining-data/sources/falcon-refinedweb/v0-0.05-heldout-complement/documents/* + attributes: + - cc_multi_bin + - dedupe_para_ngrams_13_1 + - paloma_paragraphs + - paloma_documents + - pii_regex_with_counts_fast_v2 + - tokenizer_repetitions_v2r2 + output: + max_size_in_bytes: 3_814_697_265 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1-7/documents/superhigh-25/falcon + min_text_length: 100 # at least 100 characters + discard_fields: + - attributes + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + # Contaminated Data from Paloma + - ".attributes.paloma_documents_bff_duplicates != null" + - "(.attributes.paloma_paragraphs_bff_duplicates | length) > 0" + + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + + # Quality filter + - .attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.25 + + # PII filter + - .attributes.pii_regex_with_counts_fast_v2__pii_regex_with_counts_fast_v2__doc_count[0][-1] > 5 + syntax: jq + span_replacement: + - span: "$.attributes.dedupe_para_ngrams_13_1" + min_score: 0.8 + replacement: '' + + - name: c4 + documents: + - s3://ai2-llm/pretraining-data/sources/c4/v0/documents/train/* + attributes: + - cc_multi_bin + - dedupe_para_ngrams_13_1 + - tokenizer_repetitions_v2r2 + - pii_regex_with_counts_v2 + output: + max_size_in_bytes: 2_147_483_648 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1-7/documents/superhigh-25/c4 + min_text_length: 100 # matches wikipedia + # discard_fields: + # - attributes + # - metadata + filter: + include: + - >- + (.attributes.dedupe_para_ngrams_13_1 | length == 0) or + ((.attributes.dedupe_para_ngrams_13_1 | map(.[2] * (.[1] - .[0])) | add) / (.text | length) <= 0.3) + exclude: + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + + # Quality filter + - .attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.25 + + # PII filter + - .attributes.pii_regex_with_counts_v2__pii_regex_with_counts_v2__doc_count[0][-1] > 5 + syntax: jq + span_replacement: + - span: "$.attributes.dedupe_para_ngrams_13_1" + min_score: 0.8 + replacement: '' + + - name: cc_en_head + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_docpara_le030_decontam/documents/cc_en_head/* + attributes: + - cc_multi_bin + - tokenizer_repetitions_v2r2 + output: + max_size_in_bytes: 3_894_967_296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1-7/documents/superhigh-25/cc_en_head + min_text_length: 100 # at least 100 characters + # discard_fields: + # - attributes + # - metadata + filter: + exclude: + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + + # Quality filter + - .attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.25 + syntax: jq + + + - name: cc_en_middle + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_docpara_le030_decontam/documents/cc_en_middle/* + attributes: + - cc_multi_bin + - tokenizer_repetitions_v2r2 + output: + max_size_in_bytes: 3_894_967_296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1-7/documents/superhigh-25/cc_en_middle + min_text_length: 100 # at least 100 characters + # discard_fields: + # - attributes + # - metadata + filter: + exclude: + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + + # Quality filter + - .attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.25 + syntax: jq + + + - name: cc_en_tail + documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/v1_7_dd_ngram_docpara_le030_decontam/documents/cc_en_tail/* + attributes: + - cc_multi_bin + - tokenizer_repetitions_v2r2 + output: + max_size_in_bytes: 3_894_967_296 + path: s3://ai2-llm/pretraining-data/sources/olmo-mix/v1-7/documents/superhigh-25/cc_en_tail + min_text_length: 100 # at least 100 characters + # discard_fields: + # - attributes + # - metadata + filter: + exclude: + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 10) + + # Quality filter + - .attributes.cc_multi_bin__cc_multi_bin__hq[0][-1] <= 0.25 + syntax: jq + + +work_dir: + input: "/tmp/superhigh-25/input" + output: "/tmp/superhigh-25/output" + +processes: 188 diff --git a/configs/dolma-v2/algebraic-stack_cl100k.yml b/configs/dolma-v2/algebraic-stack_cl100k.yml new file mode 100644 index 00000000..c2a5747f --- /dev/null +++ b/configs/dolma-v2/algebraic-stack_cl100k.yml @@ -0,0 +1,16 @@ +destination: ${oc.env:HOME}/ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer +documents: + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/algebraic-stack/train/* + +processes: 128 +seed: 3920 +max_size: 4_294_967_296 +dtype: uint32 + +tokenizer: + name_or_path: allenai/dolma2-tokenizer + bos_token_id: null + eos_token_id: 100257 + pad_token_id: 100277 + segment_before_tokenization: false + encode_special_tokens: true diff --git a/configs/dolma-v2/arxiv_cl100k.yml b/configs/dolma-v2/arxiv_cl100k.yml new file mode 100644 index 00000000..894075c0 --- /dev/null +++ b/configs/dolma-v2/arxiv_cl100k.yml @@ -0,0 +1,16 @@ +destination: ${oc.env:HOME}/ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer +documents: + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/arxiv/train/* + +processes: 128 +seed: 3920 +max_size: 4_294_967_296 +dtype: uint32 + +tokenizer: + name_or_path: allenai/dolma2-tokenizer + bos_token_id: null + eos_token_id: 100257 + pad_token_id: 100277 + segment_before_tokenization: false + encode_special_tokens: true diff --git a/configs/dolma-v2/dclm-long-seq.yml b/configs/dolma-v2/dclm-long-seq.yml new file mode 100644 index 00000000..c36ee417 --- /dev/null +++ b/configs/dolma-v2/dclm-long-seq.yml @@ -0,0 +1,30 @@ +streams: + - name: dclm + documents: + - s3://ai2-llm/pretraining-data/sources/dclm/v0/documents/full/*.zstd + attributes: + - tokenizer_repetitions_v2r2 + output: + max_size_in_bytes: 3_814_697_265 + path: s3://ai2-llm/pretraining-data/sources/dclm/v0_repetitions/documents/full + discard_fields: + - attributes + + compression: + input: zst + output: zst + + filter: + exclude: + # Remove repetitions + - >- + (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition != null) and (.attributes.tokenizer_repetitions_v2r2__tokenizer_repetitions_v2r2__doc_max_score_repetition[0][-1] > 32) + + syntax: jq + span_replacement: [] + +work_dir: + input: "/home/ubuntu/dclm_v0_repetitions/input" + output: "/home/ubuntu/dclm_v0_repetitions/output" + +processes: 188 diff --git a/configs/dolma-v2/dclm_cl100k.yml b/configs/dolma-v2/dclm_cl100k.yml new file mode 100644 index 00000000..1feb4e5d --- /dev/null +++ b/configs/dolma-v2/dclm_cl100k.yml @@ -0,0 +1,16 @@ +destination: ${oc.env:HOME}/ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer +documents: + - s3://ai2-llm/pretraining-data/sources/dclm/raw/hero-run-fasttext_for_HF/filtered/OH_eli5_vs_rw_v2_bigram_200k_train/fasttext_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/processed_data/global-shard_*_of_10/local-shard_*_of_10/* + +processes: 128 +seed: 3920 +max_size: 4_294_967_296 +dtype: uint32 + +tokenizer: + name_or_path: allenai/dolma2-tokenizer + bos_token_id: null + eos_token_id: 100257 + pad_token_id: 100277 + segment_before_tokenization: false + encode_special_tokens: true diff --git a/configs/dolma-v2/exp/new-sc.yaml b/configs/dolma-v2/exp/new-sc.yaml new file mode 100644 index 00000000..264b5002 --- /dev/null +++ b/configs/dolma-v2/exp/new-sc.yaml @@ -0,0 +1,1352 @@ +run_name: ${oc.env:SLURM_JOB_ID} +seed: 6198 +dry_run: false +no_pre_train_checkpoint: true + +wandb: + name: ${run_name} + project: olmoe + group: null + +model: + d_model: 2048 + n_heads: 16 + n_layers: 16 + mlp_ratio: 8 + weight_tying: false + alibi: false + rope: true + flash_attention: true + attention_dropout: 0.0 + attention_layer_norm: false + include_bias: false + block_type: sequential + layer_norm_type: rms + layer_norm_with_affine: true + bias_for_layer_norm: false + attention_layer_norm_with_affine: false + activation_type: swiglu + residual_dropout: 0.0 + embedding_dropout: 0.0 + max_sequence_length: 4096 + vocab_size: 50280 + embedding_size: 50304 + eos_token_id: 0 + pad_token_id: 1 + init_device: meta + init_fn: normal + init_std: 0.02 + init_cutoff_factor: 3 + +compile: null + +optimizer: + name: adamw + learning_rate: 4.0e-4 + eps: 1.0e-8 + weight_decay: 0.1 + decay_norm_and_bias: true + decay_embeddings: false + betas: + - 0.9 + - 0.95 + metrics_log_interval: 10 + +# Cx1: t_max = 1.3B params * 20 = 26e9 +# Cx2: t_max = 1.3B params * 40 = 52e9 +# Cx3: t_max = 1.3B params * 60 = 78e9 +# Cx4: t_max = 1.3B params * 80 = 104e9 +# Cx5: t_max = 1.3B params * 100 = 130e9 +scheduler: + name: cosine_with_warmup + units: tokens + t_warmup: 8388608000 + t_max: 52e9 + alpha_f: 0.1 + +tokenizer: + identifier: allenai/dolma2-tokenizer + truncate_direction: right + +save_folder: runs/${run_name} +remote_save_folder: s3://ai2-llm/checkpoints/OLMoE/${run_name} +save_overwrite: false + +save_interval: 5000 +save_interval_ephemeral: null +save_num_checkpoints_to_keep: -1 +sharded_checkpointer: olmo_core + +save_interval_unsharded: null +save_num_unsharded_checkpoints_to_keep: -1 + +load_path: null + +max_duration: ${scheduler.t_max} +stop_at: 12_400 +global_train_batch_size: 1024 +device_train_microbatch_size: 8 + +precision: amp_bf16 +distributed_strategy: fsdp + +fsdp: + wrapping_strategy: by_block + precision: mixed + sharding_strategy: FULL_SHARD + +max_grad_norm: 1.0 +max_grad_norm_ratio: null + +speed_monitor: + window_size: 1 + +eval_interval: 1000 +eval_subset_num_batches: -1 +device_eval_batch_size: ${device_train_microbatch_size} +evaluators: + - label: all-small-ppl-validation + data: + num_workers: 0 + drop_last: true + datasets: + c4_en-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/c4_en/val/part-0-00000.npy + dolma_books-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_books/val/part-0-00000.npy + dolma_common-crawl-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_common-crawl/val/part-0-00000.npy + dolma_pes2o-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_pes2o/val/part-0-00000.npy + dolma_reddit-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_reddit/val/part-0-00000.npy + dolma_stack-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_stack/val/part-0-00000.npy + dolma_wiki-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_wiki/val/part-0-00000.npy + ice-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/ice/val/part-0-00000.npy + m2d2_s2orc-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/m2d2_s2orc/val/part-0-00000.npy + pile-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/pile/val/part-0-00000.npy + wikitext_103-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/wikitext_103/val/part-0-00000.npy + + ########################## + # Downstream evaluations # + ########################## + - label: piqa + type: downstream + + - label: hellaswag + type: downstream + + - label: winogrande + type: downstream + + - label: openbook_qa + type: downstream + + - label: boolq + type: downstream + + - label: sciq + type: downstream + + - label: arc_easy + type: downstream + + - label: arc_challenge + type: downstream + + - label: copa + type: downstream + + - label: commonsense_qa + type: downstream + + - label: social_iqa + type: downstream + + - label: mmlu_stem_var + type: downstream + + - label: mmlu_humanities_var + type: downstream + + - label: mmlu_social_sciences_var + type: downstream + + - label: mmlu_other_var + type: downstream + + - label: mmlu_stem_mc_5shot + type: downstream + + - label: mmlu_humanities_mc_5shot + type: downstream + + - label: mmlu_social_sciences_mc_5shot + type: downstream + + - label: mmlu_other_mc_5shot + type: downstream + + - label: mmlu_stem_mc_5shot_test + type: downstream + + - label: mmlu_humanities_mc_5shot_test + type: downstream + + - label: mmlu_social_sciences_mc_5shot_test + type: downstream + + - label: mmlu_other_mc_5shot_test + type: downstream + +data: + pad_direction: right + num_workers: 16 + memmap_dtype: uint32 + drop_last: true + pin_memory: true + prefetch_factor: 8 + persistent_workers: true + timeout: 0 + paths: + # ProofPile 2: Algebraic Stack Data + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-00-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-01-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-02-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-03-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-04-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-06-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-07-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-08-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-09-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-10-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-11-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-12-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-13-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-14-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-15-00000.npy + + # ProofPile 2: Arxiv Data + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-00-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-01-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-02-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-03-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-04-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-05-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-06-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-07-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-08-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-09-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-10-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-11-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-13-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-14-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-15-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-16-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-17-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-18-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-19-00000.npy + + # ProofPile 2: Open Web Math Data + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + + # Pes2o Data + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-00-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-01-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-02-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-03-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-04-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-05-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-06-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-07-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-08-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-10-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-11-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-12-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-13-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-14-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-15-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-16-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-17-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-18-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-19-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-20-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-21-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-22-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-23-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-24-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-25-00000.npy + + # Starcoder Data (fixed!) + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-000-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-001-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-002-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-003-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-004-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-005-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-006-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-007-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-008-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-009-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-010-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-011-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-012-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-013-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-014-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-015-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-016-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-017-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-018-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-019-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-020-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-021-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-022-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-023-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-024-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-025-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-026-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-027-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-028-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-029-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-030-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-031-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-032-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-033-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-034-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-035-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-036-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-037-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-038-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-039-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-040-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-041-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-042-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-043-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-044-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-045-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-046-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-047-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-048-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-049-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-050-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-051-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-052-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-053-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-054-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-055-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-056-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-057-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-058-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-059-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-060-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-061-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-062-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-063-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-064-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-065-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-066-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-067-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-068-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-069-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-070-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-071-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-072-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-073-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-074-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-075-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-076-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-077-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-078-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-079-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-080-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-081-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-082-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-083-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-084-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-085-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-086-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-087-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-088-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-089-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-090-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-091-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-092-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-093-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-094-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-095-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-096-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-097-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-098-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer/part-099-00000.npy + + # DCLM Data + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-000-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-000-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-000-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-000-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-000-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-001-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-001-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-001-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-001-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-001-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-002-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-002-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-002-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-002-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-002-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-003-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-003-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-003-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-003-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-003-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-004-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-004-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-004-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-004-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-004-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-005-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-005-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-005-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-005-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-005-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-006-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-006-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-006-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-006-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-006-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-007-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-007-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-007-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-007-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-007-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-009-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-009-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-009-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-009-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-009-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-010-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-010-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-010-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-010-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-010-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-011-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-011-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-011-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-011-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-011-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-012-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-012-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-012-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-012-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-012-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-013-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-013-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-013-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-013-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-013-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-015-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-015-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-015-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-015-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-015-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-017-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-017-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-017-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-017-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-017-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-018-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-018-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-018-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-018-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-018-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-019-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-019-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-019-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-019-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-019-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-020-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-020-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-020-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-020-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-020-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-021-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-021-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-021-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-021-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-021-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-022-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-022-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-022-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-022-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-022-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-023-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-023-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-023-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-023-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-023-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-025-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-025-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-025-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-025-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-025-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-026-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-026-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-026-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-026-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-026-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-028-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-028-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-028-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-028-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-028-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-029-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-029-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-029-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-029-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-029-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-032-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-032-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-032-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-032-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-032-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-033-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-033-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-033-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-033-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-033-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-034-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-034-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-034-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-034-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-034-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-035-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-035-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-035-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-035-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-035-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-038-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-038-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-038-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-038-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-038-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-039-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-039-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-039-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-039-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-039-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-040-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-040-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-040-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-040-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-040-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-041-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-041-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-041-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-041-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-041-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-042-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-042-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-042-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-042-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-042-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-044-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-044-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-044-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-044-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-044-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-045-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-045-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-045-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-045-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-045-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-046-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-046-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-046-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-046-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-046-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-047-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-047-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-047-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-047-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-047-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-048-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-048-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-048-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-048-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-048-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-049-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-049-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-049-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-049-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-049-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-050-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-050-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-050-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-050-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-050-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-051-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-051-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-051-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-051-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-051-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-052-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-052-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-052-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-052-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-052-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-053-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-053-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-053-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-053-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-053-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-054-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-054-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-054-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-054-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-054-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-055-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-055-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-055-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-055-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-055-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-056-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-056-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-056-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-056-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-056-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-057-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-057-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-057-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-057-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-057-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-059-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-059-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-059-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-059-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-059-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-060-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-060-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-060-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-060-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-060-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-061-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-061-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-061-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-061-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-061-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-062-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-062-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-062-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-062-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-062-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-063-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-063-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-063-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-063-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-063-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-064-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-064-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-064-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-064-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-064-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-065-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-065-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-065-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-065-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-065-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-066-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-066-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-066-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-066-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-066-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-067-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-067-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-067-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-067-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-067-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-068-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-068-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-068-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-068-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-068-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-070-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-070-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-070-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-070-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-070-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-072-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-072-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-072-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-072-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-072-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-074-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-074-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-074-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-074-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-074-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-075-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-075-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-075-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-075-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-075-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-076-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-076-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-076-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-076-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-076-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-077-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-077-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-077-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-077-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-077-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-079-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-079-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-079-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-079-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-079-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-080-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-080-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-080-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-080-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-080-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-081-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-081-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-081-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-081-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-081-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-082-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-082-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-082-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-082-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-082-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-083-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-083-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-083-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-083-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-083-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-084-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-084-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-084-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-084-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-084-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-085-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-085-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-085-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-085-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-085-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-086-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-086-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-086-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-086-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-086-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-087-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-087-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-087-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-087-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-087-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-088-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-088-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-088-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-088-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-088-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-089-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-089-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-089-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-089-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-089-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-091-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-091-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-091-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-091-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-091-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-092-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-092-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-092-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-092-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-092-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-093-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-093-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-093-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-093-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-093-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-094-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-094-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-094-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-094-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-094-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-095-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-095-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-095-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-095-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-095-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-096-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-096-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-096-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-096-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-096-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-097-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-097-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-097-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-097-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-097-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-098-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-098-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-098-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-098-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-098-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-099-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-099-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-099-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-099-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-099-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-100-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-100-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-100-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-100-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-100-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-101-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-101-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-101-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-101-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-101-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-102-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-102-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-102-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-102-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-102-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-103-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-103-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-103-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-103-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-103-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00000 + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00001 + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00002 + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00003 + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00004 + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-106-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-106-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-106-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-106-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-106-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-108-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-108-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-108-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-108-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-108-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-109-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-109-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-109-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-109-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-109-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-111-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-111-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-111-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-111-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-111-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-112-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-112-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-112-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-112-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-112-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-113-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-113-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-113-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-113-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-113-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-115-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-115-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-115-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-115-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-115-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-116-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-116-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-116-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-116-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-116-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-119-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-119-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-119-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-119-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-119-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-120-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-120-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-120-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-120-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-120-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-121-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-121-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-121-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-121-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-121-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-122-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-122-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-122-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-122-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-122-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-123-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-123-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-123-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-123-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-123-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-126-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-126-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-126-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-126-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-126-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-127-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-127-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-127-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-127-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-127-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-128-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-128-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-128-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-128-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-128-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-129-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-129-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-129-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-129-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-129-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-131-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-131-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-131-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-131-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-131-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-132-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-132-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-132-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-132-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-132-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-133-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-133-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-133-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-133-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-133-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-134-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-134-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-134-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-134-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-134-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-136-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-136-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-136-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-136-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-136-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-137-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-137-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-137-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-137-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-137-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-138-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-138-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-138-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-138-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-138-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-139-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-139-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-139-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-139-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-139-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-141-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-141-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-141-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-141-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-141-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-142-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-142-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-142-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-142-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-142-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-143-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-143-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-143-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-143-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-143-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-144-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-144-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-144-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-144-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-144-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-145-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-145-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-145-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-145-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-145-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-146-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-146-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-146-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-146-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-146-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-147-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-147-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-147-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-147-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-147-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-148-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-148-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-148-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-148-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-148-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-149-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-149-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-149-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-149-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-149-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-150-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-150-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-150-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-150-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-150-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-151-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-151-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-151-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-151-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-151-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-153-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-153-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-153-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-153-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-153-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-155-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-155-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-155-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-155-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-155-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-156-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-156-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-156-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-156-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-156-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-157-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-157-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-157-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-157-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-157-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-158-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-158-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-158-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-158-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-158-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-159-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-159-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-159-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-159-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-159-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-161-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-161-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-161-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-161-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-161-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-163-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-163-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-163-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-163-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-163-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-164-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-164-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-164-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-164-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-164-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-165-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-165-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-165-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-165-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-165-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-166-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-166-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-166-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-166-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-166-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-167-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-167-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-167-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-167-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-167-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-168-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-168-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-168-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-168-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-168-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-169-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-169-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-169-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-169-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-169-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-172-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-172-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-172-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-172-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-172-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-173-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-173-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-173-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-173-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-173-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-175-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-175-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-175-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-175-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-175-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-177-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-177-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-177-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-177-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-177-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-178-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-178-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-178-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-178-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-178-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-179-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-179-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-179-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-179-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-179-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-180-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-180-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-180-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-180-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-180-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-181-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-181-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-181-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-181-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-181-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-182-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-182-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-182-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-182-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-182-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-183-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-183-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-183-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-183-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-183-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-184-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-184-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-184-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-184-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-184-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-185-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-185-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-185-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-185-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-185-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-187-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-187-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-187-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-187-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-187-00004.npy + + # Wikipedia + - s3://ai2-llm/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-0-00000.npy + - s3://ai2-llm/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.npy diff --git a/configs/dolma-v2/exp/old-sc.yaml b/configs/dolma-v2/exp/old-sc.yaml new file mode 100644 index 00000000..a6c2db3e --- /dev/null +++ b/configs/dolma-v2/exp/old-sc.yaml @@ -0,0 +1,1326 @@ +run_name: ${oc.env:SLURM_JOB_ID} +seed: 6198 +dry_run: false +no_pre_train_checkpoint: true + +wandb: + name: ${run_name} + project: olmoe + group: null + +model: + d_model: 2048 + n_heads: 16 + n_layers: 16 + mlp_ratio: 8 + weight_tying: false + alibi: false + rope: true + flash_attention: true + attention_dropout: 0.0 + attention_layer_norm: false + include_bias: false + block_type: sequential + layer_norm_type: rms + layer_norm_with_affine: true + bias_for_layer_norm: false + attention_layer_norm_with_affine: false + activation_type: swiglu + residual_dropout: 0.0 + embedding_dropout: 0.0 + max_sequence_length: 4096 + vocab_size: 50280 + embedding_size: 50304 + eos_token_id: 0 + pad_token_id: 1 + init_device: meta + init_fn: normal + init_std: 0.02 + init_cutoff_factor: 3 + +compile: null + +optimizer: + name: adamw + learning_rate: 4.0e-4 + eps: 1.0e-8 + weight_decay: 0.1 + decay_norm_and_bias: true + decay_embeddings: false + betas: + - 0.9 + - 0.95 + metrics_log_interval: 10 + +# Cx1: t_max = 1.3B params * 20 = 26e9 +# Cx2: t_max = 1.3B params * 40 = 52e9 +# Cx3: t_max = 1.3B params * 60 = 78e9 +# Cx4: t_max = 1.3B params * 80 = 104e9 +# Cx5: t_max = 1.3B params * 100 = 130e9 +scheduler: + name: cosine_with_warmup + units: tokens + t_warmup: 8388608000 + t_max: 52e9 + alpha_f: 0.1 + +tokenizer: + identifier: allenai/dolma2-tokenizer + truncate_direction: right + +save_folder: runs/${run_name} +remote_save_folder: s3://ai2-llm/checkpoints/OLMoE/${run_name} +save_overwrite: false + +save_interval: 5000 +save_interval_ephemeral: null +save_num_checkpoints_to_keep: -1 +sharded_checkpointer: olmo_core + +save_interval_unsharded: null +save_num_unsharded_checkpoints_to_keep: -1 + +load_path: null + +max_duration: ${scheduler.t_max} +stop_at: 12_400 +global_train_batch_size: 1024 +device_train_microbatch_size: 8 + +precision: amp_bf16 +distributed_strategy: fsdp + +fsdp: + wrapping_strategy: by_block + precision: mixed + sharding_strategy: FULL_SHARD + +max_grad_norm: 1.0 +max_grad_norm_ratio: null + +speed_monitor: + window_size: 1 + +eval_interval: 1000 +eval_subset_num_batches: -1 +device_eval_batch_size: ${device_train_microbatch_size} +evaluators: + - label: all-small-ppl-validation + data: + num_workers: 0 + drop_last: true + datasets: + c4_en-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/c4_en/val/part-0-00000.npy + dolma_books-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_books/val/part-0-00000.npy + dolma_common-crawl-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_common-crawl/val/part-0-00000.npy + dolma_pes2o-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_pes2o/val/part-0-00000.npy + dolma_reddit-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_reddit/val/part-0-00000.npy + dolma_stack-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_stack/val/part-0-00000.npy + dolma_wiki-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/dolma_wiki/val/part-0-00000.npy + ice-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/ice/val/part-0-00000.npy + m2d2_s2orc-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/m2d2_s2orc/val/part-0-00000.npy + pile-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/pile/val/part-0-00000.npy + wikitext_103-validation: + - s3://ai2-llm/eval-data/perplexity/v3_small_dolma2-tokenizer/wikitext_103/val/part-0-00000.npy + + ########################## + # Downstream evaluations # + ########################## + - label: piqa + type: downstream + + - label: hellaswag + type: downstream + + - label: winogrande + type: downstream + + - label: openbook_qa + type: downstream + + - label: boolq + type: downstream + + - label: sciq + type: downstream + + - label: arc_easy + type: downstream + + - label: arc_challenge + type: downstream + + - label: copa + type: downstream + + - label: commonsense_qa + type: downstream + + - label: social_iqa + type: downstream + + - label: mmlu_stem_var + type: downstream + + - label: mmlu_humanities_var + type: downstream + + - label: mmlu_social_sciences_var + type: downstream + + - label: mmlu_other_var + type: downstream + + - label: mmlu_stem_mc_5shot + type: downstream + + - label: mmlu_humanities_mc_5shot + type: downstream + + - label: mmlu_social_sciences_mc_5shot + type: downstream + + - label: mmlu_other_mc_5shot + type: downstream + + - label: mmlu_stem_mc_5shot_test + type: downstream + + - label: mmlu_humanities_mc_5shot_test + type: downstream + + - label: mmlu_social_sciences_mc_5shot_test + type: downstream + + - label: mmlu_other_mc_5shot_test + type: downstream + +data: + pad_direction: right + num_workers: 16 + memmap_dtype: uint32 + drop_last: true + pin_memory: true + prefetch_factor: 8 + persistent_workers: true + timeout: 0 + paths: + # ProofPile 2: Algebraic Stack Data + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-00-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-01-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-02-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-03-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-04-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-05-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-06-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-07-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-08-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-09-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-10-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-11-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-12-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-13-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-14-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/allenai/dolma2-tokenizer/part-15-00000.npy + + # ProofPile 2: Arxiv Data + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-00-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-01-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-02-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-03-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-04-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-05-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-06-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-07-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-08-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-09-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-10-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-11-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-12-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-13-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-14-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-15-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-16-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-17-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-18-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/arxiv/train/allenai/dolma2-tokenizer/part-19-00000.npy + + # ProofPile 2: Open Web Math Data + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-00-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-01-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-02-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-03-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-04-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-05-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-06-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-07-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-08-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-09-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-10-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-11-00000.npy + - s3://ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/gpt-neox-olmo-dolma-v1_5/part-12-00000.npy + + # Pes2o Data + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-00-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-01-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-02-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-03-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-04-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-05-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-06-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-07-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-08-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-09-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-10-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-11-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-12-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-13-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-14-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-15-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-16-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-17-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-18-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-19-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-20-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-21-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-22-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-23-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-24-00000.npy + - s3://ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer/part-25-00000.npy + + # Starcoder Data + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-00-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-01-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-01-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-02-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-02-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-03-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-03-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-04-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-05-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-05-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-06-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-06-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-07-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-08-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-08-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-09-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-09-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-10-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-11-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-11-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-12-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-12-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-13-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-14-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-15-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-15-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-16-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-17-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-18-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-19-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-19-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-20-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-20-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-21-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-22-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-22-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-23-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-23-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-24-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-25-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-25-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-26-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-27-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-28-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-28-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-29-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-29-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-30-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-31-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-31-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-32-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-32-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-33-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-33-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-34-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-34-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-35-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-35-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-36-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-37-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-37-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-38-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-39-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-40-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-40-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-41-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-41-00001.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-42-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-43-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-44-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-45-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-46-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-47-00000.npy + - s3://ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer/part-48-00000.npy + + # DCLM Data + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-000-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-000-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-000-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-000-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-000-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-001-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-001-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-001-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-001-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-001-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-002-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-002-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-002-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-002-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-002-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-003-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-003-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-003-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-003-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-003-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-004-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-004-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-004-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-004-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-004-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-005-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-005-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-005-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-005-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-005-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-006-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-006-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-006-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-006-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-006-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-007-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-007-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-007-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-007-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-007-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-008-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-009-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-009-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-009-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-009-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-009-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-010-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-010-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-010-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-010-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-010-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-011-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-011-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-011-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-011-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-011-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-012-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-012-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-012-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-012-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-012-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-013-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-013-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-013-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-013-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-013-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-014-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-015-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-015-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-015-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-015-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-015-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-016-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-017-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-017-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-017-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-017-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-017-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-018-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-018-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-018-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-018-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-018-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-019-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-019-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-019-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-019-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-019-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-020-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-020-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-020-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-020-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-020-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-021-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-021-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-021-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-021-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-021-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-022-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-022-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-022-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-022-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-022-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-023-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-023-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-023-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-023-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-023-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-024-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-025-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-025-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-025-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-025-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-025-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-026-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-026-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-026-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-026-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-026-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-027-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-028-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-028-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-028-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-028-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-028-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-029-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-029-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-029-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-029-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-029-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-030-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-031-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-032-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-032-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-032-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-032-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-032-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-033-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-033-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-033-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-033-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-033-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-034-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-034-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-034-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-034-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-034-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-035-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-035-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-035-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-035-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-035-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-036-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-037-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-038-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-038-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-038-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-038-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-038-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-039-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-039-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-039-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-039-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-039-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-040-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-040-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-040-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-040-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-040-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-041-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-041-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-041-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-041-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-041-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-042-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-042-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-042-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-042-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-042-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-043-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-044-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-044-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-044-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-044-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-044-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-045-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-045-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-045-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-045-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-045-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-046-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-046-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-046-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-046-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-046-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-047-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-047-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-047-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-047-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-047-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-048-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-048-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-048-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-048-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-048-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-049-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-049-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-049-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-049-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-049-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-050-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-050-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-050-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-050-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-050-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-051-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-051-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-051-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-051-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-051-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-052-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-052-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-052-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-052-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-052-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-053-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-053-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-053-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-053-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-053-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-054-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-054-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-054-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-054-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-054-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-055-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-055-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-055-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-055-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-055-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-056-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-056-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-056-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-056-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-056-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-057-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-057-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-057-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-057-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-057-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-058-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-059-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-059-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-059-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-059-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-059-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-060-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-060-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-060-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-060-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-060-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-061-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-061-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-061-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-061-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-061-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-062-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-062-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-062-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-062-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-062-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-063-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-063-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-063-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-063-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-063-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-064-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-064-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-064-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-064-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-064-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-065-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-065-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-065-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-065-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-065-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-066-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-066-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-066-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-066-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-066-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-067-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-067-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-067-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-067-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-067-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-068-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-068-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-068-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-068-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-068-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-069-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-070-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-070-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-070-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-070-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-070-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-071-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-072-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-072-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-072-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-072-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-072-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-073-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-074-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-074-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-074-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-074-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-074-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-075-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-075-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-075-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-075-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-075-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-076-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-076-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-076-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-076-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-076-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-077-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-077-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-077-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-077-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-077-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-078-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-079-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-079-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-079-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-079-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-079-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-080-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-080-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-080-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-080-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-080-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-081-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-081-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-081-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-081-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-081-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-082-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-082-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-082-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-082-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-082-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-083-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-083-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-083-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-083-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-083-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-084-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-084-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-084-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-084-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-084-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-085-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-085-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-085-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-085-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-085-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-086-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-086-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-086-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-086-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-086-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-087-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-087-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-087-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-087-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-087-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-088-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-088-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-088-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-088-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-088-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-089-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-089-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-089-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-089-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-089-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-090-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-091-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-091-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-091-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-091-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-091-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-092-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-092-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-092-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-092-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-092-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-093-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-093-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-093-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-093-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-093-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-094-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-094-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-094-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-094-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-094-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-095-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-095-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-095-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-095-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-095-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-096-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-096-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-096-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-096-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-096-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-097-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-097-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-097-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-097-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-097-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-098-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-098-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-098-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-098-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-098-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-099-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-099-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-099-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-099-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-099-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-100-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-100-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-100-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-100-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-100-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-101-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-101-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-101-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-101-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-101-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-102-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-102-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-102-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-102-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-102-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-103-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-103-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-103-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-103-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-103-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00000 + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00001 + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00002 + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00003 + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-104-00004 + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-105-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-106-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-106-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-106-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-106-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-106-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-107-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-108-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-108-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-108-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-108-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-108-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-109-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-109-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-109-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-109-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-109-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-110-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-111-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-111-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-111-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-111-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-111-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-112-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-112-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-112-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-112-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-112-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-113-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-113-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-113-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-113-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-113-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-114-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-115-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-115-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-115-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-115-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-115-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-116-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-116-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-116-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-116-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-116-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-117-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-118-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-119-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-119-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-119-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-119-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-119-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-120-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-120-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-120-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-120-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-120-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-121-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-121-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-121-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-121-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-121-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-122-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-122-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-122-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-122-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-122-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-123-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-123-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-123-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-123-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-123-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-124-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-125-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-126-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-126-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-126-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-126-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-126-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-127-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-127-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-127-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-127-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-127-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-128-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-128-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-128-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-128-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-128-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-129-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-129-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-129-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-129-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-129-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-130-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-131-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-131-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-131-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-131-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-131-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-132-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-132-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-132-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-132-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-132-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-133-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-133-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-133-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-133-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-133-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-134-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-134-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-134-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-134-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-134-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-135-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-136-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-136-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-136-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-136-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-136-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-137-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-137-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-137-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-137-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-137-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-138-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-138-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-138-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-138-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-138-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-139-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-139-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-139-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-139-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-139-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-140-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-141-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-141-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-141-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-141-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-141-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-142-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-142-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-142-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-142-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-142-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-143-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-143-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-143-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-143-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-143-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-144-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-144-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-144-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-144-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-144-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-145-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-145-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-145-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-145-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-145-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-146-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-146-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-146-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-146-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-146-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-147-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-147-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-147-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-147-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-147-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-148-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-148-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-148-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-148-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-148-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-149-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-149-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-149-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-149-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-149-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-150-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-150-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-150-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-150-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-150-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-151-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-151-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-151-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-151-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-151-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-152-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-153-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-153-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-153-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-153-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-153-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-154-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-155-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-155-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-155-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-155-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-155-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-156-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-156-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-156-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-156-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-156-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-157-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-157-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-157-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-157-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-157-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-158-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-158-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-158-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-158-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-158-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-159-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-159-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-159-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-159-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-159-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-160-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-161-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-161-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-161-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-161-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-161-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-162-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-163-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-163-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-163-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-163-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-163-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-164-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-164-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-164-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-164-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-164-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-165-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-165-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-165-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-165-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-165-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-166-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-166-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-166-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-166-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-166-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-167-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-167-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-167-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-167-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-167-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-168-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-168-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-168-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-168-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-168-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-169-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-169-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-169-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-169-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-169-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-170-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-171-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-172-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-172-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-172-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-172-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-172-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-173-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-173-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-173-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-173-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-173-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-174-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-175-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-175-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-175-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-175-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-175-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-176-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-177-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-177-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-177-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-177-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-177-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-178-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-178-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-178-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-178-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-178-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-179-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-179-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-179-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-179-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-179-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-180-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-180-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-180-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-180-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-180-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-181-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-181-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-181-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-181-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-181-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-182-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-182-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-182-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-182-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-182-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-183-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-183-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-183-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-183-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-183-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-184-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-184-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-184-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-184-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-184-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-185-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-185-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-185-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-185-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-185-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-186-00004.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-187-00000.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-187-00001.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-187-00002.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-187-00003.npy + - s3://ai2-llm/preprocessed/dclm/text_openhermes_reddit_eli5_vs_rw_v2_bigram_200k_train/allenai/dolma2-tokenizer/part-187-00004.npy + + # Wikipedia + - s3://ai2-llm/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-0-00000.npy + - s3://ai2-llm/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer/part-1-00000.npy diff --git a/configs/dolma-v2/owm_cl100k.yml b/configs/dolma-v2/owm_cl100k.yml new file mode 100644 index 00000000..41cf7f0a --- /dev/null +++ b/configs/dolma-v2/owm_cl100k.yml @@ -0,0 +1,16 @@ +destination: ${oc.env:HOME}/ai2-llm/preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/allenai/dolma2-tokenizer +documents: + - s3://ai2-llm/pretraining-data/sources/proof-pile-2/v0_decontaminated/documents/open-web-math/train/* + +processes: 128 +seed: 3920 +max_size: 4_294_967_296 +dtype: uint32 + +tokenizer: + name_or_path: allenai/dolma2-tokenizer + bos_token_id: null + eos_token_id: 100257 + pad_token_id: 100277 + segment_before_tokenization: false + encode_special_tokens: true diff --git a/configs/dolma-v2/pes2o_cl100k.yml b/configs/dolma-v2/pes2o_cl100k.yml new file mode 100644 index 00000000..c20e4483 --- /dev/null +++ b/configs/dolma-v2/pes2o_cl100k.yml @@ -0,0 +1,16 @@ +destination: ${oc.env:HOME}/ai2-llm/preprocessed/pes2o/allenai/dolma2-tokenizer +documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/danyh-compiled-v1_7/documents/pes2o/* + +processes: 128 +seed: 3920 +max_size: 4_294_967_296 +dtype: uint32 + +tokenizer: + name_or_path: allenai/dolma2-tokenizer + bos_token_id: null + eos_token_id: 100257 + pad_token_id: 100277 + segment_before_tokenization: false + encode_special_tokens: true diff --git a/configs/dolma-v2/starcoder-v0_cl100k.yml b/configs/dolma-v2/starcoder-v0_cl100k.yml new file mode 100644 index 00000000..33faba37 --- /dev/null +++ b/configs/dolma-v2/starcoder-v0_cl100k.yml @@ -0,0 +1,16 @@ +destination: ${oc.env:HOME}/ai2-llm/preprocessed/starcoder/v0_decontaminated_doc_only/allenai/dolma2-tokenizer +documents: + - s3://ai2-llm/pretraining-data/sources/starcoder/v0_decontaminated_doc_only/documents/* + +processes: 128 +seed: 3920 +max_size: 4_294_967_296 +dtype: uint32 + +tokenizer: + name_or_path: allenai/dolma2-tokenizer + bos_token_id: null + eos_token_id: 100257 + pad_token_id: 100277 + segment_before_tokenization: false + encode_special_tokens: true diff --git a/configs/dolma-v2/starcoder-v1_cl100k.yml b/configs/dolma-v2/starcoder-v1_cl100k.yml new file mode 100644 index 00000000..a6b100f5 --- /dev/null +++ b/configs/dolma-v2/starcoder-v1_cl100k.yml @@ -0,0 +1,16 @@ +destination: ${oc.env:HOME}/ai2-llm/preprocessed/starcoder/v1-decon-100_to_20k-2star-top_token_030/allenai/dolma2-tokenizer +documents: + - s3://ai2-llm/pretraining-data/sources/starcoder/v1-decon-100_to_20k-2star-top_token_030/documents/* + +processes: 128 +seed: 3920 +max_size: 4_294_967_296 +dtype: uint32 + +tokenizer: + name_or_path: allenai/dolma2-tokenizer + bos_token_id: null + eos_token_id: 100257 + pad_token_id: 100277 + segment_before_tokenization: false + encode_special_tokens: true diff --git a/configs/dolma-v2/tokenize_eval.sh b/configs/dolma-v2/tokenize_eval.sh new file mode 100644 index 00000000..a56d4837 --- /dev/null +++ b/configs/dolma-v2/tokenize_eval.sh @@ -0,0 +1,62 @@ +tokenizer="allenai/dolma2-tokenizer" +path="s3://ai2-llm/eval-data/perplexity" +suffix="dolma2-tokenizer" + + +v2_eval=( + "4chan" + "c4_100_domains" + "c4_en" + "gab" + "ice" + "m2d2_s2orc" + "m2d2_wiki" + "manosphere" + "mc4_en" + "pile" + "ptb" + "twitterAEE" + "wikitext_103" +) + +v3_eval=( + "c4_en" + "dolma_books" + "dolma_common-crawl" + "dolma_pes2o" + "dolma_reddit" + "dolma_stack" + "dolma_wiki" + "ice" + "m2d2_s2orc" + "pile" + "wikitext_103" +) + +set -ex + +for dataset in "${v2_eval[@]}"; do + for split in "val" "test"; do + dolma tokens \ + --tokenizer.name_or_path $tokenizer \ + --tokenizer.eos_token_id 100257 \ + --tokenizer.pad_token_id 100277 \ + --tokenizer.encode_special_tokens \ + --dtype uint32 \ + --destination "${path}/v2_small_${suffix}/${dataset}/${split}" \ + --documents "${path}/v2_small/${dataset}/${split}/*.gz" & + done +done + +for dataset in "${v3_eval[@]}"; do + for split in "val" "test"; do + dolma tokens \ + --tokenizer.name_or_path $tokenizer \ + --tokenizer.eos_token_id 100257 \ + --tokenizer.pad_token_id 100277 \ + --tokenizer.encode_special_tokens \ + --dtype uint32 \ + --destination "${path}/v3_small_${suffix}/${dataset}/${split}" \ + --documents "${path}/v3_small/${dataset}/${split}/*.gz" & + done +done diff --git a/configs/dolma-v2/wiki_cl100k.yml b/configs/dolma-v2/wiki_cl100k.yml new file mode 100644 index 00000000..cf8fb725 --- /dev/null +++ b/configs/dolma-v2/wiki_cl100k.yml @@ -0,0 +1,16 @@ +destination: ${oc.env:HOME}/ai2-llm/preprocessed/olmo-mix/danyh-compiled-v1_7/documents/wiki/allenai/dolma2-tokenizer +documents: + - s3://ai2-llm/pretraining-data/sources/olmo-mix/danyh-compiled-v1_7/documents/wiki/* + +processes: 128 +seed: 3920 +max_size: 4_294_967_296 +dtype: uint32 + +tokenizer: + name_or_path: allenai/dolma2-tokenizer + bos_token_id: null + eos_token_id: 100257 + pad_token_id: 100277 + segment_before_tokenization: false + encode_special_tokens: true diff --git a/pyproject.toml b/pyproject.toml index dc89ab78..7df69af6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "anyascii>=0.3.2", "blingfire==0.1.8", "boto3>=1.28", + # "boto3[crt]>=1.33", "cached-path>=1.5.1", # "fasttext==0.9.2", # broken with new version of setuptools; using fasttext-wheel instead "fasttext-wheel==0.9.2", @@ -18,6 +19,7 @@ dependencies = [ "omegaconf>=2.3.0", # "pycld2==0.41", # "pycld3==0.22", # does not install correctly + "hyperscan>=0.7.0", "platformdirs>=4.2.0", "pyyaml", "requests", @@ -30,7 +32,8 @@ dependencies = [ "numpy<2", "necessary>=0.4.3", "charset-normalizer>=3.2.0", - "zstandard>=0.23.0", + "backoff>=2.0.0", + "zstandard>=0.23.0" ] classifiers = [ "Development Status :: 5 - Production/Stable", @@ -100,7 +103,7 @@ dolma = "dolma.cli.__main__:main" [project.optional-dependencies] dev = [ - "black>=22.6.0", + "black[jupyter]>=22.6.0", "flake8>=5.0", "flake8-pyi>=22.8.1", "Flake8-pyproject>=1.1.0", @@ -124,7 +127,13 @@ lang = [ ] # extension to parse warc files -warc = ["fastwarc", "w3lib", "url-normalize"] +warc = [ + "fastwarc", + "w3lib", + "courlan", + "dateparser", +] + trafilatura = [ # must include warc dependencies "dolma[warc]", @@ -140,6 +149,11 @@ trafilatura = [ resiliparse = ["dolma[warc]", "resiliparse"] +openwebmath = [ + "dolma[warc]", + "openwebmath-text-extract", +] + # all extensions all = [ "dolma[dev]", @@ -169,7 +183,7 @@ features = ["pyo3/extension-module"] where = ["src"] [tool.setuptools.package-data] -dolma = ["py.typed", "data/*"] +dolma = ["py.typed", "data/*", "*.pyi"] [tool.black] line-length = 115 diff --git a/python/dolma/__init__.py b/python/dolma/__init__.py index f0da3ad0..17f79e2f 100644 --- a/python/dolma/__init__.py +++ b/python/dolma/__init__.py @@ -9,7 +9,7 @@ # must import taggers to register them # we import the rust extension here and wrap it in a python module from . import dolma as _dolma # type: ignore # noqa: E402 -from .core import TaggerRegistry # noqa: E402 +from .core import TaggerRegistry, add_compression # noqa: E402 from .core.errors import DolmaRustPipelineError # noqa: E402 from .core.taggers import BaseTagger # noqa: E402 from .taggers import * # noqa: E402 diff --git a/python/dolma/cli/__init__.py b/python/dolma/cli/__init__.py index 2ef221d7..a5fdf45b 100644 --- a/python/dolma/cli/__init__.py +++ b/python/dolma/cli/__init__.py @@ -25,6 +25,7 @@ get_origin, ) +from necessary import necessary from omegaconf import MISSING, DictConfig, ListConfig from omegaconf import OmegaConf as om from omegaconf.errors import OmegaConfBaseException @@ -33,13 +34,7 @@ from ..core.errors import DolmaConfigError -__all__ = [ - "BaseCli", - "field", - "make_parser", - "namespace_to_nested_omegaconf", - "print_config", -] +__all__ = ["BaseCli", "field", "make_parser", "namespace_to_nested_omegaconf", "print_config"] T = TypeVar("T", bound=Any) @@ -137,6 +132,15 @@ def _make_nested_dict(key: str, value: Any, d: Optional[Dict[str, Any]] = None) return d +def to_native_types( + obj: Any, resolve: bool = True, throw_on_missing: bool = True, enum_to_str: bool = True +) -> Any: + """Converts an OmegaConf object to native types (dicts, lists, etc.)""" + if isinstance(obj, DictConfig) or isinstance(obj, ListConfig): + return om.to_container(obj, resolve=resolve, throw_on_missing=throw_on_missing, enum_to_str=enum_to_str) + return obj + + def namespace_to_nested_omegaconf(args: Namespace, structured: Type[T], config: Optional[dict] = None) -> T: nested_config_dict: Dict[str, Any] = {} for key, value in vars(args).items(): @@ -146,7 +150,13 @@ def namespace_to_nested_omegaconf(args: Namespace, structured: Type[T], config: om.create(config or {}), om.create(nested_config_dict) ) # pyright: ignore (pylance is confused because om.create might return a DictConfig or a ListConfig) + # resolve any interpolations in the config + om.resolve(untyped_config) + + # create structured config from cli dataclass base_structured_config: DictConfig = om.structured(structured) + + # merge with options parsed from config file and merged_config = om.merge(base_structured_config, untyped_config) # check for type @@ -199,3 +209,65 @@ def run_from_args(cls, args: Namespace, config: Optional[dict] = None): @classmethod def run(cls, parsed_config: D): raise NotImplementedError("Abstract method; must be implemented in subclass") + + +def patch_old_omegaconf(): + """Monkey patch omegaconf below version 2.3.0 to support custom resolver returning + lists or dicts. Applies patch https://github.com/omry/omegaconf/pull/1093""" + + if necessary(("omegaconf", "2.4.0"), soft=True): + # no need to patch + return + + if getattr(patch_old_omegaconf, "__patched__", False): + # already patched + return + + from omegaconf import _impl # pylint: disable=import-outside-toplevel + from omegaconf import ( # pylint: disable=import-outside-toplevel + Container, + Node, + ValueNode, + ) + from omegaconf._utils import ( # noqa: F401 # pylint: disable=import-outside-toplevel + _ensure_container, + _get_value, + is_primitive_container, + is_structured_config, + ) + from omegaconf.errors import ( # pylint: disable=import-outside-toplevel + InterpolationToMissingValueError, + ) + from omegaconf.nodes import ( # pylint: disable=import-outside-toplevel + InterpolationResultNode, + ) + + def _resolve_container_value(cfg: Container, key: Any) -> None: + node = cfg._get_child(key) # pylint: disable=protected-access + assert isinstance(node, Node) + if node._is_interpolation(): # pylint: disable=protected-access + try: + resolved = node._dereference_node() # pylint: disable=protected-access + except InterpolationToMissingValueError: + node._set_value(MISSING) # pylint: disable=protected-access + else: + if isinstance(resolved, Container): + _impl._resolve(resolved) # pylint: disable=protected-access + if isinstance(resolved, InterpolationResultNode): + resolved_value = _get_value(resolved) + if is_primitive_container(resolved_value) or is_structured_config(resolved_value): + resolved = _ensure_container(resolved_value) + if isinstance(resolved, Container) and isinstance(node, ValueNode): + cfg[key] = resolved + else: + node._set_value(_get_value(resolved)) # pylint: disable=protected-access + else: + _impl._resolve(node) # pylint: disable=protected-access + + # set new function and mark as patched + setattr(_impl, "_resolve_container_value", _resolve_container_value) + setattr(patch_old_omegaconf, "__patched__", True) + + +# actually executes the patch +patch_old_omegaconf() diff --git a/python/dolma/cli/__main__.py b/python/dolma/cli/__main__.py index 55f14f44..e196a171 100644 --- a/python/dolma/cli/__main__.py +++ b/python/dolma/cli/__main__.py @@ -1,4 +1,94 @@ -from dolma.cli.main import main +import multiprocessing +from argparse import ArgumentParser +from pathlib import Path +from typing import Any, Dict, List, Optional, Union -if __name__ == "__main__": - main() +import smart_open +from yaml import safe_load + +from ..core.paths import exists +from .analyzer import AnalyzerCli +from .deduper import DeduperCli +from .listers import ListerCli +from .mixer import MixerCli + +# must import these to register the resolvers +from .resolvers import * # noqa: F401,F403,W0401 +from .tagger import TaggerCli +from .tokenizer import TokenizerCli +from .warc import WarcExtractorCli + +AVAILABLE_COMMANDS = { + "dedupe": DeduperCli, + "mix": MixerCli, + "tag": TaggerCli, + "list": ListerCli, + "stat": AnalyzerCli, + "tokens": TokenizerCli, + "warc": WarcExtractorCli, + # following functionality is not yet implemented + # "train-ft": None, + # "train-lm": None, +} + + +def read_config(path: Union[None, str]) -> Dict[str, Any]: + """Read a configuration file if it exists""" + if path is None: + return {} + + if not exists(path): + raise FileNotFoundError(f"Config file {path} does not exist") + + with smart_open.open(path, mode="rt") as f: + return dict(safe_load(f)) + + +def main(argv: Optional[List[str]] = None): + """Main entry point for the CLI""" + + try: + # attempting to set start method to spawn in case it is not set + multiprocessing.set_start_method("spawn") + except RuntimeError as ex: + # method already set, check if it is set to spawn + if multiprocessing.get_start_method() != "spawn": + raise RuntimeError("Multiprocessing start method must be set to spawn") from ex + + parser = ArgumentParser( + prog="dolma", + usage="dolma {global options} [command] {command options}", + description="Command line interface for the Dolma processing toolkit", + ) + parser.add_argument( + "-c", + "--config", + help="Path to configuration optional file", + type=Path, + default=None, + ) + + # Continue by adding subparsers and parsing the arguments + subparsers = parser.add_subparsers(dest="command") + subparsers.required = True + subparsers.choices = AVAILABLE_COMMANDS.keys() # type: ignore + for command, cli in AVAILABLE_COMMANDS.items(): + cli.make_parser(subparsers.add_parser(command, help=cli.DESCRIPTION)) + + # parse the arguments + args = parser.parse_args(argv) + + # first, get the command and config path to run + command = args.__dict__.pop("command") + config_path = args.__dict__.pop("config", None) or None + + # remove the other optional arguments from the top level parser + args.__dict__.pop("dolma_version", None) + args.__dict__.pop("dolma_commands", None) + + # read the config file if one was provided + config = read_config(config_path) + + # get the cli for the command and run it with the config we just loaded + the args + cli = AVAILABLE_COMMANDS[command] + return cli.run_from_args(args=args, config=config) diff --git a/python/dolma/cli/analyzer.py b/python/dolma/cli/analyzer.py index 414615b5..c78738e3 100644 --- a/python/dolma/cli/analyzer.py +++ b/python/dolma/cli/analyzer.py @@ -8,6 +8,17 @@ from dolma.core.loggers import get_logger from dolma.core.paths import glob_path +@dataclass +class BinsConfig: + compute: int = field( + default=1_000, + help="Number of bins to use to compute the histograms.", + ) + visualization: int = field( + default=10, + help="Number of bins to use when visualizing the histograms.", + ) + @dataclass class AnalyzerConfig: @@ -22,9 +33,9 @@ class AnalyzerConfig: "If not provided, the report will be printed to stdout." ), ) - bins: int = field( - default=1_000, - help="Number of bins to use for the histograms.", + bins: BinsConfig = field( + default=BinsConfig(), + help="Configuration for the bins to use for the histograms." ) processes: int = field( default=1, @@ -80,7 +91,8 @@ def run(cls, parsed_config: AnalyzerConfig): metadata_path=work_dirs.input, debug=parsed_config.debug, seed=parsed_config.seed, - num_bins=parsed_config.bins, + compute_bins=parsed_config.bins.compute, + visualize_bins=parsed_config.bins.visualization, num_processes=parsed_config.processes, name_regex=parsed_config.regex, show_total=parsed_config.total, diff --git a/python/dolma/cli/listers.py b/python/dolma/cli/listers.py new file mode 100644 index 00000000..623eb83b --- /dev/null +++ b/python/dolma/cli/listers.py @@ -0,0 +1,70 @@ +from dataclasses import dataclass +from typing import List, Optional + +from rich.console import Console +from rich.table import Table + +from dolma.cli import BaseCli, field +from dolma.core.loggers import get_logger +from dolma.core.registry import BaseRegistry +from dolma.core.utils import import_modules + + +@dataclass +class ListerConfig: + modules: List[str] = field( + default=[], + help="List of Python modules $PYTHONPATH to import custom registry modules from.", + ) + tagger_modules: Optional[List[str]] = field( + default=None, + help="List of Python modules $PYTHONPATH to import custom taggers from.", + ) + filter: Optional[str] = field( + default=None, + help="Filter which registries to list.", + ) + + +class ListerCli(BaseCli): + CONFIG = ListerConfig + DESCRIPTION = "List all available modules in registry." + + @classmethod + def run(cls, parsed_config: ListerConfig): + if parsed_config.tagger_modules is not None: + # deprecation warning + logger = get_logger(__file__) + logger.warning( + "The `tagger_modules` argument is deprecated and will be removed in a future release. " + "Please use `modules` instead." + ) + parsed_config.modules.extend(parsed_config.tagger_modules) + + # import tagger modules + import_modules(parsed_config.modules) + + for reg_item_name, reg_item_cls in BaseRegistry.registries(): + if parsed_config.filter is not None and parsed_config.filter.lower() not in reg_item_name.lower(): + continue + + any_has_description = any( + reg_item_desc for _, _, reg_item_desc in reg_item_cls.items_with_description() + ) + + table = Table(title=reg_item_name, style="bold") + table.width + table.add_column("name", justify="left", style="cyan", no_wrap=True, ratio=1) + table.add_column("class", justify="left", style="magenta", no_wrap=False, ratio=1) + if any_has_description: + table.add_column("description", justify="left", style="blue", no_wrap=False, ratio=4) + + for reg_item_name, reg_item_cls, reg_item_desc in sorted(reg_item_cls.items_with_description()): + registry_module = f"{reg_item_cls.__module__}.{reg_item_cls.__name__}" + if any_has_description: + table.add_row(reg_item_name, registry_module, reg_item_desc) + else: + table.add_row(reg_item_name, registry_module) + + console = Console() + console.print(table) diff --git a/python/dolma/cli/mixer.py b/python/dolma/cli/mixer.py index 943d7f74..6a2efad4 100644 --- a/python/dolma/cli/mixer.py +++ b/python/dolma/cli/mixer.py @@ -77,6 +77,10 @@ class MixerConfig: default=False, help="If true, only print the configuration and exit without running the mixer.", ) + skip_checks: bool = field( + default=False, + help="If true, skip checks on paths (e.g. validation, globbing). Useful in case many paths are being evaluated.", + ) class MixerCli(BaseCli): @@ -141,19 +145,22 @@ def run(cls, parsed_config: MixerConfig): if "span_replacement" not in stream_config_dict and "filter" not in stream_config_dict: raise DolmaConfigError("Either `filter` or `span_replacement` must be specified") - # perform some path validation to make sure we don't call the mixer with invalid config - total_matching_documents = 0 - for document in stream_config.documents: + if not parsed_config.skip_checks: + # perform some path validation to make sure we don't call the mixer with invalid config + total_matching_documents = 0 + for document in stream_config.documents: - current_matching_documents = sum(1 for _ in glob_path(document)) - if current_matching_documents == 0: - # only raise a warning if no documents are found for a single path - logger.warning("No documents found for path %s", document) - total_matching_documents += current_matching_documents + current_matching_documents = sum(1 for _ in glob_path(document)) + if current_matching_documents == 0: + # only raise a warning if no documents are found for a single path + logger.warning("No documents found for path %s", document) + total_matching_documents += current_matching_documents - if total_matching_documents == 0: - # but raise an error if no documents are found for all paths - raise DolmaConfigError(f"No documents found for the paths for {stream_config.name} config.") + if total_matching_documents == 0: + # but raise an error if no documents are found for all paths + raise DolmaConfigError( + f"No documents found for the paths for {stream_config.name} config." + ) # populate the stream config dict stream_config_dict["name"] = stream_config.name diff --git a/python/dolma/cli/resolvers.py b/python/dolma/cli/resolvers.py index ad799619..e1869a5d 100644 --- a/python/dolma/cli/resolvers.py +++ b/python/dolma/cli/resolvers.py @@ -1,36 +1,77 @@ import multiprocessing -from typing import List, TypeVar +import re +import sys +from typing import Callable, List, Optional, TypeVar, Union -from cached_path import cached_path +import smart_open from omegaconf.omegaconf import OmegaConf as om -from omegaconf.omegaconf import Resolver -from ..core.paths import glob_path +from ..core.paths import cached_path, glob_path +from ..core.registry import BaseRegistry -__all__ = ["cache", "glob", "processes"] +C = TypeVar("C", bound=Callable) -C = TypeVar("C", bound=Resolver) +class ResolverRegistry(BaseRegistry[Callable]): + @classmethod + def add(cls, name: str, desc: Optional[str] = None) -> Callable[[C], C]: + _add_fn = super().add(name, desc) + def _wrapped_add_fn( + resolver: C, + base_add_fn: C = _add_fn, # type: ignore + resolver_name: str = name, + ) -> C: + base_add_fn(resolver) + om.register_new_resolver(resolver_name, resolver, replace=True) + return resolver -def resolver(resolver: C) -> C: - resolver_name = f"d.{resolver.__name__}" - om.register_new_resolver(resolver_name, resolver, replace=True) - return resolver + return _wrapped_add_fn -@resolver +@ResolverRegistry.add("d.cache", "Download a file and replace the path with the cached path.") def cache(path: str) -> str: return str(cached_path(path)) -@resolver +@ResolverRegistry.add("d.glob", "Glob this path and return a list of files.") def glob(path: str) -> List[str]: globbed = list(glob_path(path)) assert len(globbed) > 0, f"Path {path} does not match any files" return globbed -@resolver +@ResolverRegistry.add("d.procs", "Return the number of processes available (optionally with buffer).") def processes(n: int = 0) -> int: return max(1, multiprocessing.cpu_count() - n) + + +@ResolverRegistry.add("d.stdin", "Read from stdin and return list of lines.") +def stdin() -> List[str]: + return [stripped_line for line in sys.stdin if (stripped_line := line.strip())] + + +@ResolverRegistry.add("d.unescape", "Unescape a Unicode sequence.") +def unescape(expr: str) -> str: + return expr.encode().decode("unicode_escape") + + +@ResolverRegistry.add("d.file", "Read from a file and return contents.") +def file_(path: str, mode: str = "rt", encoding: str = "utf-8") -> str: + with smart_open.open(path, mode=mode, encoding=encoding) as f: + return str(f.read()) + + +@ResolverRegistry.add("d.sed", "Perform a sed-like substitution on a string.") +def sed(data: Union[str, List[str]], expression: str, separator: str = "/") -> Union[str, List[str]]: + if isinstance(data, list): + return [str(sed(d, expression, separator)) for d in data] + + pattern, replacement = expression.strip(separator).split(separator) + modified_data = re.sub(pattern, replacement, data) + return modified_data + + +@ResolverRegistry.add("d.split", "Split string into list of strings on symbol.") +def split(string: str, symbol: str = "\n") -> List[str]: + return [stripped_line for line in string.split(symbol) if (stripped_line := line.strip())] diff --git a/python/dolma/cli/shared.py b/python/dolma/cli/shared.py index 3ef47dec..51491a66 100644 --- a/python/dolma/cli/shared.py +++ b/python/dolma/cli/shared.py @@ -1,10 +1,11 @@ import copy import os +import sys import tempfile from contextlib import ExitStack, contextmanager from dataclasses import dataclass from pathlib import Path -from typing import Generator, Optional +from typing import Generator, List, Optional, Union from dolma.cli import field @@ -45,3 +46,17 @@ def make_workdirs(config: WorkDirConfig) -> Generator[WorkDirConfig, None, None] config.output = stack.enter_context(tempfile.TemporaryDirectory(prefix="dolma-output-")) yield config + + +def maybe_parse_from_stdin(paths: Union[str, List[str]]) -> List[str]: + """ + If paths is a single string equal to "-", read from stdin and return a list of lines; + otherwise return the input as is. + """ + if isinstance(paths, str): + paths = [paths] + + if paths == ["-"]: + return [str(r.strip()) for r in sys.stdin] + + return paths[:] diff --git a/python/dolma/cli/tagger.py b/python/dolma/cli/tagger.py index 9982ec05..86fe70d4 100644 --- a/python/dolma/cli/tagger.py +++ b/python/dolma/cli/tagger.py @@ -2,17 +2,12 @@ from pstats import SortKey from typing import List, Optional -from rich.console import Console -from rich.table import Table - from dolma.cli import BaseCli, field, print_config -from dolma.cli.shared import WorkDirConfig, make_workdirs +from dolma.cli.shared import CompressionConfig, WorkDirConfig, make_workdirs from dolma.core.errors import DolmaConfigError from dolma.core.loggers import get_logger from dolma.core.paths import glob_path -from dolma.core.registry import TaggerRegistry from dolma.core.runtime import create_and_run_tagger -from dolma.core.utils import import_modules @dataclass @@ -82,6 +77,13 @@ class TaggerConfig: default=False, help="Whether to run in debug mode.", ) + compression: CompressionConfig = field( + default=CompressionConfig(), + help=( + "Compression configuration for input and output; if not provided, " + "compression will inferred from file extension" + ), + ) profile: ProfilerConfig = field( default=ProfilerConfig(), help="Whether to run in profiling mode.", @@ -140,33 +142,6 @@ def run(cls, parsed_config: TaggerConfig): profile_output=parsed_config.profile.output, profile_steps=parsed_config.profile.steps, profile_sort_key=parsed_config.profile.sort_key, + compression_input=parsed_config.compression.input, + compression_output=parsed_config.compression.output, ) - - -@dataclass -class ListTaggerConfig: - tagger_modules: List[str] = field( - default=[], - help="List of Python modules $PYTHONPATH to import custom taggers from.", - ) - - -class ListTaggerCli(BaseCli): - CONFIG = ListTaggerConfig - DESCRIPTION = "List available taggers." - - @classmethod - def run(cls, parsed_config: ListTaggerConfig): - # import tagger modules - import_modules(parsed_config.tagger_modules) - - table = Table(title="dolma taggers", style="bold") - table.add_column("name", justify="left", style="cyan") - table.add_column("class", justify="left", style="magenta") - - for tagger_name, tagger_cls in sorted(TaggerRegistry.items()): - tagger_repr = f"{tagger_cls.__module__}.{tagger_cls.__name__}" - table.add_row(tagger_name, tagger_repr) - - console = Console() - console.print(table) diff --git a/python/dolma/cli/warc.py b/python/dolma/cli/warc.py index 9a8da2ea..6593270e 100644 --- a/python/dolma/cli/warc.py +++ b/python/dolma/cli/warc.py @@ -1,7 +1,7 @@ from dataclasses import dataclass -from typing import List +from typing import List, Optional -from dolma.cli import BaseCli, field, print_config +from dolma.cli import BaseCli, field, print_config, to_native_types from dolma.cli.shared import WorkDirConfig, make_workdirs from dolma.core.errors import DolmaConfigError from dolma.core.loggers import get_logger @@ -19,6 +19,37 @@ class TaggerConfig: default=False, help="Whether to skip if taggers returns no output.", ) + mode: str = field( + default="any", + help=( + "Mode to use for taggers: If 'any', any tagger output is enough; " + "if 'all', all taggers must output. Ignore if skip is False. Defaults to 'any'." + ), + ) + + +@dataclass +class BackoffConfig: + max_time: Optional[float] = field( + default=None, + help="Maximum time to wait between retries in seconds.", + ) + max_tries: int = field( + default=10, + help="Maximum number of retries.", + ) + + +@dataclass +class MetadataStorageConfig: + html: bool = field( + default=False, + help="Whether to store the HTML content in the metadata.", + ) + attr_spans: int = field( + default=-1, + help="Whether to store the spans corresponding to attributes in the metadata.", + ) @dataclass @@ -48,23 +79,54 @@ class WarcExtractorConfig: default=False, help="Whether to run in debug mode.", ) + compression: str = field( + default="zst", + help="Compression to use for the outputs. Defaults to zst. Must be supported in smart_open library.", + ) source_name: str = field(help="Name to assign to the source.") linearizer: str = field( default="resiliparse", help="Name of the HTML linearizer to use.", ) + fast_linearizer: Optional[str] = field( + default=None, + help=( + "If provided, use this linearizer to get approximate representation of document " + "to cheaply run post taggers. If not provide, the regular linearizer will be used." + ), + ) + min_raw_length: int = field( + default=0, + help="Minimum length of HTML content to process.", + ) + min_text_length: int = field( + default=0, + help="Minimum length of text content to process.", + ) pre: TaggerConfig = field(default=TaggerConfig(), help="Configuration for pre-extraction taggers.") post: TaggerConfig = field(default=TaggerConfig(), help="Configuration for post-extraction taggers.") - store_html_in_metadata: bool = field( - default=False, - help="Whether to store the HTML content in the metadata.", + store: MetadataStorageConfig = field( + default=MetadataStorageConfig(), help="Configuration for metadata storage." + ) + delay_start: float = field( + default=0, + help="Delay in seconds to wait before starting the pipeline. Random delay is picked from 0 to this value.", ) - work_dir: WorkDirConfig = field(default=WorkDirConfig(), help="Configuration for temporary work directories.") dryrun: bool = field( default=False, - help="If true, only print the configuration and exit without running the taggers.", + help="If true, only print the configuration and exit without running the pipieline.", + ) + skip_checks: bool = field( + default=False, + help="If true, skip checks on paths (e.g. validation, globbing). Useful in case many paths are being evaluated.", ) + skip_duplicate_urls: bool = field( + default=False, + help="If true, skip documents with duplicate URLs within a single process.", + ) + batch_size: int = field(default=1, help="Number of files to process into a single output file.") + backoff: BackoffConfig = field(default=BackoffConfig(), help="Configuration for backoff retries.") class WarcExtractorCli(BaseCli): @@ -83,19 +145,20 @@ def run(cls, parsed_config: WarcExtractorConfig): if not isinstance(source_name, str): raise ValueError(f"source_name must be a string, not {source_name} ({type(source_name)})") - # perform some path validation to make sure we don't call - # the extractor with invalid config - total_matching_documents = 0 - for document in documents: - current_matching_documents = sum(1 for _ in glob_path(document)) - if current_matching_documents == 0: - # only raise a warning if no documents are found for a single path - logger.warning("No documents found for path %s", document) - total_matching_documents += current_matching_documents - - if total_matching_documents == 0: - # but raise an error if no documents are found for all paths - raise DolmaConfigError(f"No documents found for paths {documents}.") + if not parsed_config.skip_checks: + # perform some path validation to make sure we don't call the warc + # extractor with an invalid config + total_matching_documents = 0 + for document in documents: + current_matching_documents = sum(1 for _ in glob_path(document)) + if current_matching_documents == 0: + # only raise a warning if no documents are found for a single path + logger.warning("No documents found for path %s", document) + total_matching_documents += current_matching_documents + + if total_matching_documents == 0: + # but raise an error if no documents are found for all paths + raise DolmaConfigError(f"No documents found for paths {documents}.") print_config(parsed_config) if parsed_config.dryrun: @@ -103,17 +166,30 @@ def run(cls, parsed_config: WarcExtractorConfig): return create_and_run_warc_pipeline( - documents=(documents[0] if len(documents) == 1 else documents), - destination=(destination[0] if len(destination) == 1 else destination), + documents=to_native_types(documents[0] if len(documents) == 1 else documents), + destination=to_native_types(destination[0] if len(destination) == 1 else destination), metadata=work_dirs.output, num_processes=parsed_config.processes, ignore_existing=parsed_config.ignore_existing, debug=parsed_config.debug, source_name=source_name, - pre_taggers=parsed_config.pre.taggers, + compression=parsed_config.compression, + pre_taggers=to_native_types(parsed_config.pre.taggers), + pre_taggers_mode=parsed_config.pre.mode, skip_no_pre_taggers=parsed_config.pre.skip, - post_taggers=parsed_config.post.taggers, + post_taggers=to_native_types(parsed_config.post.taggers), + post_taggers_mode=parsed_config.post.mode, skip_no_post_taggers=parsed_config.post.skip, - store_html_in_metadata=parsed_config.store_html_in_metadata, + store_html_in_metadata=parsed_config.store.html, + store_attribute_spans_in_metadata=parsed_config.store.attr_spans, linearizer_name=parsed_config.linearizer, + fast_linearizer_name=parsed_config.fast_linearizer, + skip_source_glob=parsed_config.skip_checks, + skip_duplicate_urls=parsed_config.skip_duplicate_urls, + backoff_max_time=parsed_config.backoff.max_time, + backoff_max_tries=parsed_config.backoff.max_tries, + batch_size=parsed_config.batch_size, + min_raw_length=parsed_config.min_raw_length, + min_text_length=parsed_config.min_text_length, + delay_start=parsed_config.delay_start, ) diff --git a/python/dolma/core/__init__.py b/python/dolma/core/__init__.py index 9219cd50..d27ea07d 100644 --- a/python/dolma/core/__init__.py +++ b/python/dolma/core/__init__.py @@ -2,6 +2,9 @@ from .registry import TaggerRegistry from .taggers import BaseTagger +# importing utils to make sure that decompressors for smart_open are registered +from .utils import add_compression # noqa: F401 + __all__ = [ "BaseTagger", "DocResult", diff --git a/python/dolma/core/analyzer.py b/python/dolma/core/analyzer.py index c8d542c0..8368ee73 100644 --- a/python/dolma/core/analyzer.py +++ b/python/dolma/core/analyzer.py @@ -283,7 +283,8 @@ def create_and_run_analyzer( report: Optional[str] = None, debug: bool = False, seed: int = 0, - num_bins: int = 1000, + compute_bins: int = 1_000, + visualize_bins: int = 10, num_processes: int = 1, name_regex: Optional[str] = None, show_total: bool = False, @@ -300,7 +301,8 @@ def create_and_run_analyzer( report (Optional[str], optional): Path to the report directory. Defaults to None. debug (bool, optional): Enable debug mode. Defaults to False. seed (int, optional): Seed value for randomization. Defaults to 0. - num_bins (int, optional): Number of bins for analysis. Defaults to 1000. + compute_bins (int, optional): Number of bins for analysis. Defaults to 1_000. + visualize_bins (int, optional): Number of bins for visualization. Defaults to 10. num_processes (int, optional): Number of processes to use for analysis. Defaults to 1. name_regex (Optional[str], optional): Regular expression for filtering attribute names. Defaults to None. show_total (bool, optional): Show total summary. Defaults to False. @@ -325,11 +327,11 @@ def create_and_run_analyzer( debug=debug, seed=seed, ignore_existing=True, - retries_on_error=0, + backoff_max_tries=1, num_processes=num_processes, ) - analyzer(num_bins=num_bins, name_regex=name_regex) + analyzer(num_bins=compute_bins, name_regex=name_regex) - summaries = aggregate_summaries(summaries_path=summaries_path, num_bins=num_bins) - visualize_summaries(summaries=summaries, show_total=show_total) + summaries = aggregate_summaries(summaries_path=summaries_path, num_bins=compute_bins) + visualize_summaries(summaries=summaries, show_total=show_total, num_viz_bins=visualize_bins) write_output(summaries=summaries, report=report) diff --git a/python/dolma/core/binning.py b/python/dolma/core/binning.py index 3afee92f..c33fd0da 100644 --- a/python/dolma/core/binning.py +++ b/python/dolma/core/binning.py @@ -205,7 +205,8 @@ def merge_bins( class BaseBucketApi: def __init__(self): - self._total = self._sum = 0 + self._total = 0 + self._sum = 0.0 @abstractproperty def full(self) -> bool: diff --git a/python/dolma/core/data_types.py b/python/dolma/core/data_types.py index d71bbab3..c1a0a7a7 100644 --- a/python/dolma/core/data_types.py +++ b/python/dolma/core/data_types.py @@ -6,10 +6,13 @@ """ -from typing import Any, Dict, List, Optional, Tuple +import functools +import re +from hashlib import sha1 +from typing import Any, Callable, Dict, List, Optional, Tuple, Type from msgspec import Struct -from typing_extensions import TypeAlias +from typing_extensions import Self, TypeAlias TaggerOutputValueType: TypeAlias = Tuple[int, int, float] TaggerOutputType: TypeAlias = List[TaggerOutputValueType] @@ -17,13 +20,18 @@ class InputSpec(Struct): - id: str text: str + id: str = "" source: str = "" - created: str = "" - added: str = "" + created: Optional[Any] = "" + added: Optional[Any] = "" version: Optional[str] = None + def __post_init__(self): + if not self.id: + (h := sha1()).update(self.text.encode()) + self.id = h.hexdigest() + class InputSpecWithMetadata(InputSpec): metadata: Optional[Dict[str, Any]] = None @@ -40,27 +48,38 @@ class OutputSpec(Struct): class Document: - __slots__ = "source", "version", "id", "text" + __slots__ = "source", "version", "id", "text", "added", "created" + spec_cls: Type[InputSpec] = InputSpec - def __init__(self, source: str, id: str, text: str, version: Optional[str] = None) -> None: + def __init__( + self, + source: str, + id: str, + text: str, + version: Optional[str] = None, + added: Optional[str] = None, + created: Optional[str] = None, + ) -> None: self.source = source self.version = version self.id = id self.text = text + self.added = added + self.created = created @classmethod - def from_spec(cls, spec: InputSpec) -> "Document": - return Document(source=spec.source, version=spec.version, id=spec.id, text=spec.text) + def from_spec(cls, spec: InputSpec) -> Self: + return cls(**{k: v for k in cls.__slots__ if (v := getattr(spec, k)) is not None}) def to_spec(self) -> InputSpec: - return InputSpec(source=self.source, version=self.version, id=self.id, text=self.text) + return self.spec_cls(**{k: v for k in self.__slots__ if (v := getattr(self, k)) is not None}) @classmethod - def from_json(cls, d: Dict[str, Any]) -> "Document": - return Document(source=d["source"], version=d["version"], id=d["id"], text=d["text"]) + def from_json(cls, d: Dict[str, Any]) -> Self: + return cls(**{k: v for k in cls.__slots__ if (v := d.get(k)) is not None}) def to_json(self) -> Dict[str, Any]: - return {"source": self.source, "version": self.version, "id": self.id, "text": self.text} + return {k: v for k in self.__slots__ if (v := getattr(self, k)) is not None} def __str__(self) -> str: attributes_string = ",".join([f"{k}:{repr(v)}" for k, v in self.to_json().items()]) @@ -68,110 +87,36 @@ def __str__(self) -> str: class DocumentWithMetadata(Document): - __slots__ = ("metadata",) + __slots__ = Document.__slots__ + ("metadata",) + spec_cls = InputSpecWithMetadata def __init__(self, *args, metadata: Optional[Dict[str, Any]] = None, **kwargs) -> None: super().__init__(*args, **kwargs) self.metadata = metadata or {} - @classmethod - def from_spec(cls, spec: InputSpecWithMetadata) -> "DocumentWithMetadata": - return DocumentWithMetadata( - source=spec.source, - version=spec.version, - id=spec.id, - text=spec.text, - metadata=spec.metadata, - ) - - def to_spec(self) -> InputSpecWithMetadata: - return InputSpecWithMetadata( - source=self.source, - version=self.version, - id=self.id, - text=self.text, - metadata=self.metadata, - ) - - @classmethod - def from_json(cls, d: Dict) -> "DocumentWithMetadata": - return DocumentWithMetadata( - source=d["source"], - version=d["version"], - id=d["id"], - text=d["text"], - metadata=d["metadata"], - ) - - def to_json(self) -> Dict: - return { - "source": self.source, - "version": self.version, - "id": self.id, - "text": self.text, - "metadata": self.metadata, - } - def __str__(self) -> str: repr_ = super().__str__() return repr_.rstrip(")") + f",metadata={'...' if self.metadata else 'none'})" class DocumentWithMetadataAndAttributes(DocumentWithMetadata): + __slots__ = DocumentWithMetadata.__slots__ + ("attributes",) + spec_cls = InputSpecWithMetadataAndAttributes + def __init__( self, *args, attributes: Optional[Dict[str, List[Tuple[int, int, float]]]] = None, **kwargs ) -> None: super().__init__(*args, **kwargs) self.attributes = attributes or {} - @classmethod - def from_spec(cls, spec: InputSpecWithMetadataAndAttributes) -> "DocumentWithMetadataAndAttributes": - return DocumentWithMetadataAndAttributes( - source=spec.source, - version=spec.version, - id=spec.id, - text=spec.text, - metadata=spec.metadata, - attributes=spec.attributes, - ) - - @classmethod - def from_json(cls, d: Dict) -> "DocumentWithMetadataAndAttributes": - return DocumentWithMetadataAndAttributes( - source=d["source"], - version=d["version"], - id=d["id"], - text=d["text"], - metadata=d["metadata"], - attributes=d["attributes"], - ) - - def to_json(self) -> Dict: - return { - "source": self.source, - "version": self.version, - "id": self.id, - "text": self.text, - "metadata": self.metadata, - "attributes": self.attributes, - } - - def to_spec(self) -> InputSpecWithMetadataAndAttributes: - return InputSpecWithMetadataAndAttributes( - source=self.source, - version=self.version, - id=self.id, - text=self.text, - metadata=self.metadata, - attributes=self.attributes, - ) - def __str__(self) -> str: return super().__str__().rstrip(")") + f",attributes={'...' if self.attributes else 'none'})" class Span: - __slots__ = "start", "end", "type", "score", "experiment", "tagger" + __slots__ = "start", "end", "type", "score", "experiment", "tagger", "location" + + __selectors_cache__: Dict[str, Callable[["Document"], str]] = {} def __init__( self, @@ -181,6 +126,7 @@ def __init__( score: float = 1.0, experiment: Optional[str] = None, tagger: Optional[str] = None, + location: str = "text", ): self.start = start self.end = end @@ -188,15 +134,58 @@ def __init__( self.score = float(score) self.experiment = experiment self.tagger = tagger + self.location = location + + def _make_selector(self) -> Callable[["Document"], str]: + if self.location not in self.__selectors_cache__: + + def _nested_selector( + doc: Any, + index: Optional[int] = None, + key: Optional[str] = None, + previous: Optional[Callable] = None, + dict_like: bool = True, + ) -> Any: + prev = previous(doc) if previous is not None else doc + if dict_like or index is not None: + assert (key or index) is not None, "Either key or index must be set" + return prev[key or index] + elif key is not None: + return getattr(prev, key) + else: + raise ValueError("Either key or index must be set") + + matches = list( + re.finditer(r"((^|\.)(?P[a-zA-Z][a-zA-Z0-9]*))|(\[(?P[0-9]+)\])", self.location) + ) + assert len(matches) > 0, f"Invalid location: `{self.location}`" + init_match, *rest_matches = matches + + fn = functools.partial( + _nested_selector, + index=int(init_match.group("index")) if init_match.group("index") is not None else None, + key=init_match.group("key"), + dict_like=False, + ) + for match in rest_matches[::-1]: + fn = functools.partial( + _nested_selector, + index=int(match.group("index")) if match.group("index") is not None else None, + key=match.group("key"), + previous=fn, + ) + self.__selectors_cache__[self.location] = fn + + return self.__selectors_cache__[self.location] def mention(self, text: str, window: int = 0) -> str: return text[max(0, self.start - window) : min(len(text), self.end + window)] - def select(self, doc: Document) -> str: - return doc.text[self.start : self.end] + def select(self, doc: Document, left: int = 0, right: int = 0) -> str: + return self._make_selector()(doc)[self.start - left : self.end + right] @classmethod - def from_spec(cls, attribute_name: str, attribute_value: TaggerOutputValueType) -> "Span": + def from_spec(cls, attribute_name: str, attribute_value: TaggerOutputValueType) -> Self: if "__" in attribute_name: # bff tagger has different name exp_name, tgr_name, attr_type = attribute_name.split("__", 2) @@ -204,7 +193,7 @@ def from_spec(cls, attribute_name: str, attribute_value: TaggerOutputValueType) exp_name = tgr_name = attr_type = attribute_name start, end, score = attribute_value - return Span( + return cls( start=int(start), end=int(end), type=attr_type, @@ -214,19 +203,18 @@ def from_spec(cls, attribute_name: str, attribute_value: TaggerOutputValueType) ) def to_spec(self) -> Tuple[str, TaggerOutputValueType]: + from .utils import format_span_key, format_span_output + assert self.experiment is not None, "Experiment name must be set to convert to spec" assert self.tagger is not None, "Tagger name must be set to convert to spec" - return ( - f"{self.experiment}__{self.tagger}__{self.type}", - (self.start, self.end, self.score), - ) + return format_span_key(self.experiment, self.tagger, self), format_span_output(self) def __len__(self) -> int: return self.end - self.start @classmethod - def from_json(cls, di: Dict) -> "Span": - return Span(start=di["start"], end=di["end"], type=di["type"], score=di["score"]) + def from_json(cls, di: Dict) -> Self: + return cls(**{k: v for k, v in di.items() if k in cls.__slots__}) def to_json(self, text: Optional[str] = None, window: int = 0) -> dict: span_repr = {"start": self.start, "end": self.end, "type": self.type, "score": self.score} diff --git a/python/dolma/core/loggers.py b/python/dolma/core/loggers.py index f34ba864..f0ff05f3 100644 --- a/python/dolma/core/loggers.py +++ b/python/dolma/core/loggers.py @@ -5,15 +5,20 @@ DOLMA_PREFIX = "dolma" -def get_logger(name: str) -> logging.Logger: +def get_logger(name: str, level: Union[int, str] = logging.WARN) -> logging.Logger: if (proc_name := multiprocessing.current_process().name) == "MainProcess": proc_name = "main" proc_name = proc_name.replace(" ", "_") + # set the log level + level = level if isinstance(level, int) else getattr(logging, level.strip().upper(), logging.WARN) + + # set name name = f"{proc_name}.dolma.{name}" logger = logging.getLogger(name) - logger.setLevel(logging.WARN) + logger.setLevel(level) + # add handler if not logger.handlers: handler = logging.StreamHandler() formatter = logging.Formatter( diff --git a/python/dolma/core/mp_tools.py b/python/dolma/core/mp_tools.py new file mode 100644 index 00000000..98c8faf0 --- /dev/null +++ b/python/dolma/core/mp_tools.py @@ -0,0 +1,130 @@ +import multiprocessing +import time +from contextlib import ExitStack +from multiprocessing.managers import SyncManager +from multiprocessing.pool import Pool +from queue import Queue +from typing import Any, Callable, Dict, Generic, Iterable, Optional, TypeVar, Union + +T = TypeVar("T") +R = TypeVar("R") + + +def get_manager(pool: Union[Pool, "PoolWithDebug"]) -> Union[SyncManager, "ManagerWithDebug"]: + if getattr(pool, "debug", False): + return ManagerWithDebug() + else: + return multiprocessing.Manager() + + +class ResultWithDebug(Generic[T]): + def __init__(self, result: T, *args, **kwargs): + self.result = result + + def get(self, timeout: Optional[float] = None) -> T: + return self.result + + def wait(self, timeout: Optional[float] = None) -> None: + time.sleep(timeout or 0) + + def successful(self) -> bool: + return True + + def ready(self) -> bool: + return True + + +class ManagerWithDebug: + def Queue(self): + return Queue() + + def shutdown(self) -> None: + pass + + +class PoolWithDebug: + """A wrapper around multiprocessing.Pool that allows for debugging (i.e., running without multiprocessing). + Supports creating a manager for shared memory objects (mock in case of debugging).""" + + def __init__( + self, + processes: Optional[int] = None, + initializer: Optional[Callable[..., Any]] = None, + initargs: Iterable[Any] = (), + maxtasksperchild: Optional[int] = None, + debug: bool = False, + ): + self.processes = processes + self.initializer = initializer + self.initargs = initargs + self.maxtasksperchild = maxtasksperchild + self.debug = debug + + # we are gonna keep track of resources in stack; but also keeping them indexed + # separately for easy access + self.stack = ExitStack() + self._manager: Optional[SyncManager] = None + self._pool: Optional[Pool] = None + + # let's make sure that the start method is spawn for best performance + try: + multiprocessing.set_start_method("spawn") + except RuntimeError: + assert multiprocessing.get_start_method() == "spawn", "Multiprocessing start method must be spawn" + + def __enter__(self): + if self._pool is None and not self.debug: + self._pool = self.stack.enter_context( + Pool( + processes=self.processes, + initializer=self.initializer, + initargs=self.initargs, + maxtasksperchild=self.maxtasksperchild, + ) + ) + return self + + def Manager(self): + if self._manager is None: + self._manager = ( + ManagerWithDebug() # pyright: ignore + if self.debug + else self.stack.enter_context(multiprocessing.Manager()) + ) + return self._manager + + def __exit__(self, *exc): + return self.stack.close() + + def apply_async( + self, + func: Callable[..., R], + args: Iterable[Any] = (), + kwds: Dict[str, Any] = {}, + callback: Optional[Callable[[R], Any]] = None, + error_callback: Optional[Callable[[Any], Any]] = None, + ): + if self._pool is None: + if self.initializer: + # run the initializer once by calling it with the initargs and then setting it to None + self.initializer(*self.initargs) + self.initializer = None + try: + resp = func(*args, **kwds) + if callback is not None: + callback(resp) + return ResultWithDebug(resp) + except Exception as e: + if error_callback is not None: + error_callback(e) + raise e + else: + return self._pool.apply_async( + func=func, args=args, kwds=kwds, callback=callback, error_callback=error_callback + ) + + def close(self): + return self._pool and self._pool.close() + + def join(self): + return self._pool and self._pool.join() diff --git a/python/dolma/core/mp_tools.pyi b/python/dolma/core/mp_tools.pyi new file mode 100644 index 00000000..30fc1f2e --- /dev/null +++ b/python/dolma/core/mp_tools.pyi @@ -0,0 +1,19 @@ +from collections.abc import Callable, Iterable +from multiprocessing.managers import SyncManager +from multiprocessing.pool import ApplyResult, Pool +from typing import Any + +class ResultWithDebug(ApplyResult): ... # noqa: E701,E302 +class ManagerWithDebug(SyncManager): ... # noqa: E701 + +class PoolWithDebug(Pool): # noqa: E302 + def __init__( # noqa: E704 + self, + processes: int | None = None, + initializer: Callable[..., Any] | None = None, + initargs: Iterable[Any] = (), + maxtasksperchild: int | None = None, + debug: bool = False, + ): ... + +def get_manager(pool: Pool) -> SyncManager: ... # noqa: E701, E704, E302 diff --git a/python/dolma/core/parallel.py b/python/dolma/core/parallel.py index 0bbfc75f..dbe4a1ce 100644 --- a/python/dolma/core/parallel.py +++ b/python/dolma/core/parallel.py @@ -1,34 +1,45 @@ -import inspect import itertools import logging import multiprocessing import pickle import random import re -import time -from contextlib import ExitStack from datetime import datetime from functools import partial from queue import Queue -from threading import Thread -from typing import Any, Dict, List, NamedTuple, Optional, Tuple, TypeVar, Union +from typing import ( + Any, + Dict, + List, + Literal, + NamedTuple, + Optional, + Tuple, + Type, + TypeVar, + Union, +) +import backoff import smart_open -import tqdm +from backoff.types import Details from typing_extensions import TypeAlias from .errors import DolmaError, DolmaRetryableFailure from .loggers import get_logger +from .mp_tools import PoolWithDebug, get_manager from .paths import ( add_suffix, + exists, glob_path, join_path, make_relative, mkdir_p, parent, split_path, - sub_prefix, ) +from .progressbar import BaseProgressBar +from .utils import batch_iterator METADATA_SUFFIX = ".done.txt" @@ -45,9 +56,28 @@ class AllPathsTuple(NamedTuple): kwargs: List[KwargsType] @classmethod - def empty(cls) -> "AllPathsTuple": + def new(cls) -> "AllPathsTuple": return AllPathsTuple([], [], [], []) + def __len__(self) -> int: + return len(self.src) + + @property + def empty(self) -> bool: + return len(self.src) == 0 + + def partition(self, k: int = 1) -> List["AllPathsTuple"]: + """Partition the paths into k / n slices containing k files each.""" + return [ + AllPathsTuple( + src=self.src[i : i + k], + dst=self.dst[i : i + k], + meta=self.meta[i : i + k], + kwargs=self.kwargs[i : i + k], + ) + for i in range(0, len(self.src), k) + ] + class BaseParallelProcessor: """A base parallel processor that supports applying the same process_single method to a list of files. @@ -60,6 +90,8 @@ class BaseParallelProcessor: See documentation of both methods for more details on how to implement them correctly. """ + PROGRESS_BAR_CLS: Type[BaseProgressBar] + def __init__( self, source_prefix: Union[str, List[str]], @@ -70,11 +102,18 @@ def __init__( seed: int = 0, pbar_timeout: float = 1e-3, ignore_existing: bool = False, + skip_source_glob: bool = False, + shuffle_src_paths: bool = True, include_paths: Optional[List[str]] = None, exclude_paths: Optional[List[str]] = None, files_regex_pattern: Optional[str] = None, - retries_on_error: int = 0, + batch_size: int = 1, process_single_kwargs: Union[None, KwargsType, List[KwargsType]] = None, + backoff_max_time: Optional[float] = None, + backoff_max_tries: int = 1, + retries_on_error: Optional[int] = None, + backoff_exceptions: Optional[Union[Type[Exception], Tuple[Type[Exception], ...]]] = None, + progress_bar_mode: Literal["tqdm", "logger"] = "tqdm", ): """Initialize the parallel processor. @@ -95,22 +134,31 @@ def __init__( seed (int, optional): The random seed to use when shuffling input files. Defaults to 0. pbar_timeout (float, optional): How often to update progress bars in seconds. Defaults to 0.01 seconds. + skip_source_glob (bool, optional): Do not glob source files. Off by default. ignore_existing (bool, optional): Whether to ignore files that have been already processed and re-run the processor on all files from scratch. Defaults to False. - include_paths (Optional[List[str]], optional): A list of paths to include. If provided, only files + shuffle_src_paths (bool, optional): Whether to shuffle the source paths before processing them. + Defaults to True. + include_paths (List[str], optional): A list of paths to include. If provided, only files that match one of the paths will be processed. Defaults to None. - exclude_paths (Optional[List[str]], optional): A list of paths to exclude. If provided, files that + exclude_paths (List[str], optional): A list of paths to exclude. If provided, files that match one of the paths will be skipped. Defaults to None. - files_regex_pattern (Optional[str], optional): A regex pattern to match files. If provided, only + files_regex_pattern (str, optional): A regex pattern to match files. If provided, only files that match the pattern will be processed. Defaults to None. - retries_on_error (int, optional): The number of retries to attempt if an error occurs. - Defaults to 0. - process_single_kwargs (Union[None, KwargsType, List[KwargsType]], optional): Additional kwargs to + batch_size: (int, optional): number of files to group in a single bat + process_single_kwargs (Union[None, KwargsType, List[KwargsType], optional): Additional kwargs to pass to the process_single method. If a single dict is provided, it will be used for all source prefixes. If a list of dicts is provided, each dict will be used for the corresponding source. By default, no additional kwargs are passed. + backoff_max_time (float, optional): The maximum time to backoff. Defaults to None. + backoff_max_tries (int, optional): The maximum number of tries to backoff. Defaults to 1. + backoff_exceptions (Union[Type[Exception], Tuple[Type[Exception], ...]], optional): The + exceptions to backoff on. Defaults to `dolma.core.errors.DolmaRetryableFailure`. + retries_on_error (int, optional): Deprecated. The number of retries to attempt on error. + Defaults to None. + progress_bar_mode ("tqdm" or "logger", optional): The mode to use for the progress bar. + Defaults to "tqdm". """ - self.src_prefixes = [source_prefix] if isinstance(source_prefix, str) else source_prefix self.dst_prefixes = [destination_prefix] if isinstance(destination_prefix, str) else destination_prefix self.meta_prefixes = [metadata_prefix] if isinstance(metadata_prefix, str) else metadata_prefix @@ -119,11 +167,36 @@ def __init__( self.seed = seed self.pbar_timeout = pbar_timeout self.ignore_existing = ignore_existing + self.progress_bar_mode = progress_bar_mode + + self.logger = self.get_logger() self.include_paths = set(include_paths) if include_paths is not None else None self.exclude_paths = set(exclude_paths) if exclude_paths is not None else None self.files_regex_pattern = re.compile(files_regex_pattern) if files_regex_pattern else None - self.retries_on_error = retries_on_error + self.shuffle_src_paths = shuffle_src_paths + + # this manages how many files to pass to a single processor + self.batch_size = batch_size + + if retries_on_error is not None: + self.logger.warning( + "The `retries_on_error` parameter is deprecated and will be removed in a future release. " + "Please use `backoff_max_tries` instead." + ) + backoff_max_tries = retries_on_error + 1 + + # this controls backoff + self.backoff_max_time: float = float(backoff_max_time or "inf") + self.backoff_max_tries: int = int(backoff_max_tries) + self.backoff_exceptions: Tuple[Type[Exception], ...] = ( + (backoff_exceptions,) + if isinstance(backoff_exceptions, type) + else backoff_exceptions or (DolmaRetryableFailure,) + ) + + if progress_bar_mode not in ("tqdm", "logger"): + raise ValueError("Progress bar mode must be either 'tqdm' or 'logger'") # this are additional kwargs to pass to the process_single method process_single_kwargs = process_single_kwargs or {} @@ -132,23 +205,8 @@ def __init__( else: self.process_single_kwargs = process_single_kwargs - # checking that the increment_progressbar method is subclassed correctly - sig = inspect.signature(self.increment_progressbar) - if "queue" not in sig.parameters or sig.parameters["queue"].kind != inspect.Parameter.POSITIONAL_ONLY: - raise AttributeError( - "increment_progressbar must have a positional-only argument named 'queue'; " - "Check that you have subclassed BaseParallelProcessor correctly!" - ) - if "kwargs" in sig.parameters and sig.parameters["kwargs"].kind == inspect.Parameter.VAR_KEYWORD: - raise AttributeError( - "increment_progressbar must not have a **kwargs argument; " - "Check that you have subclassed BaseParallelProcessor correctly!" - ) - if any(p.name != "queue" and p.default != 0 for p in sig.parameters.values()): - raise AttributeError( - "increment_progressbar must have a default value of 0 for all arguments except 'queue'; " - "Check that you have subclassed BaseParallelProcessor correctly!" - ) + if not hasattr(self, "PROGRESS_BAR_CLS"): + self.PROGRESS_BAR_CLS = BaseProgressBar.from_increment_function(self) if len(self.src_prefixes) != len(self.dst_prefixes): raise ValueError( @@ -169,13 +227,75 @@ def __init__( if len(self.src_prefixes) == 0: raise ValueError("At least one source prefix must be provided.") + self.skip_source_glob = skip_source_glob + if any("*" in p for p in itertools.chain(self.dst_prefixes, self.meta_prefixes)): raise ValueError("Destination and metadata prefixes cannot contain wildcards.") + if not hasattr(self, "PROGRESS_BAR_CLS"): + raise AttributeError("BaseParallelProcessor subclasses must define the PROGRESS_BAR_CLS attribute.") + + def __add__(self: BPP, other: BPP) -> BPP: + """Combine two parallel processors into one.""" + if not type(self) is type(other): + raise TypeError(f"Cannot add {type(self)} and {type(other)}") + + # we try combining the two list of include paths; if they are both None, then set the combo back to none + include_paths: Union[List[str], None] = [*(self.include_paths or []), *(other.include_paths or [])] + include_paths = sorted(set(include_paths or [])) if len(include_paths or []) else None + + # do the same for exclude paths + exclude_paths: Union[List[str], None] = [*(self.exclude_paths or []), *(other.exclude_paths or [])] + exclude_paths = sorted(set(exclude_paths or [])) if len(exclude_paths or []) else None + + # for the regex, do a simple or if both are set + regex_pattern: Union[str, None] = None + if self.files_regex_pattern and other.files_regex_pattern: + regex_pattern = "(" + self.files_regex_pattern.pattern + "|" + other.files_regex_pattern.pattern + ")" + elif self.files_regex_pattern: + regex_pattern = self.files_regex_pattern.pattern + elif other.files_regex_pattern: + regex_pattern = other.files_regex_pattern.pattern + + return type(self)( + source_prefix=[*self.src_prefixes, *other.src_prefixes], + destination_prefix=[*self.dst_prefixes, *other.dst_prefixes], + metadata_prefix=[*self.meta_prefixes, *other.meta_prefixes], + num_processes=max(self.num_processes, other.num_processes), + debug=self.debug or other.debug, + seed=self.seed, + pbar_timeout=max(self.pbar_timeout, other.pbar_timeout), + ignore_existing=self.ignore_existing or other.ignore_existing, + include_paths=include_paths, + exclude_paths=exclude_paths, + files_regex_pattern=regex_pattern, + batch_size=max(self.batch_size, other.batch_size), + process_single_kwargs=[*self.process_single_kwargs, *other.process_single_kwargs], + backoff_max_time=min(self.backoff_max_time, other.backoff_max_time), + backoff_max_tries=min(self.backoff_max_tries, other.backoff_max_tries), + backoff_exceptions=tuple(set(self.backoff_exceptions + other.backoff_exceptions)), + ) + + def __radd__(self: BPP, other: BPP) -> BPP: + """Combine two parallel processors into one.""" + return other.__add__(self) + @classmethod def get_logger(cls) -> logging.Logger: """Get the logger for the class.""" - return get_logger(cls.__name__) + return get_logger(cls.__name__, "info") + + @classmethod + def process_batch( + cls, + source_paths: List[str], + destination_paths: List[str], + queue: QueueType, + kwargs: List[Dict[str, Any]], + ): + """Process multiple files. Naively calls process_single for each file, but can be overridden.""" + for src_path, dst_path, single_kwargs in zip(source_paths, destination_paths, kwargs): + cls.process_single(source_path=src_path, destination_path=dst_path, queue=queue, **single_kwargs) @classmethod def process_single( @@ -199,36 +319,61 @@ def process_single( raise NotImplementedError() @classmethod - def _process_single_and_save_status( + def _log_backoff(cls, details: Details): + """Log backoff details.""" + message = ( + f"Backing off `{details['target'].__name__}` " + f"after {details['tries']:,} " + f"tries (wait: {details.get('wait', 0.0):.2f}s)" + ) + if ex := details.get("exception"): + # add details about the exception to the message + import traceback # pylint: disable=import-outside-toplevel + + message += " due to " + "".join(traceback.format_exception_only(type(ex), ex)).strip() + + cls.get_logger().warning(message) + + @classmethod + def _process_batch_and_save_status( cls, - source_path: str, - destination_path: str, - metadata_path: str, + source_paths: List[str], + destination_paths: List[str], + metadata_paths: List[str], queue: QueueType, - serialized_kwargs: bytes, + serialized_kwargs: List[bytes], + backoff_max_time: float, + backoff_max_tries: int, + backoff_exceptions: Tuple[Type[Exception], ...], ): """A wrapper around process single that saves a metadata file if processing is successful.""" # make destination directory if it doesn't exist for the destination and metadata paths - mkdir_p(parent(destination_path)) - mkdir_p(parent(metadata_path)) - - kwargs = pickle.loads(serialized_kwargs) - retries_on_error = kwargs.get("retries_on_error", 0) + 1 - while True: - try: - cls.process_single( - source_path=source_path, destination_path=destination_path, queue=queue, **kwargs - ) - break - except DolmaRetryableFailure as exception: - retries_on_error -= 1 - if retries_on_error == 0: - raise DolmaError from exception + for path in itertools.chain(destination_paths, metadata_paths): + mkdir_p(parent(path)) + + # we unpickle the serialized kwargs + deserialized_kwargs = [pickle.loads(kw) for kw in serialized_kwargs] + + # use backoff library to retry on failure; function _log_backoff is called on backoff + # to inform the user of the backoff details. + fn_with_backoff = backoff.on_exception( + backoff.expo, + exception=backoff_exceptions, + max_tries=backoff_max_tries, + max_time=backoff_max_time, + on_backoff=cls._log_backoff, + )(cls.process_batch) + + # start processing the file here + fn_with_backoff( + source_paths=source_paths, destination_paths=destination_paths, queue=queue, kwargs=deserialized_kwargs + ) - # write the metadata file - with smart_open.open(metadata_path, "wt") as f: - f.write(datetime.now().isoformat()) + # write the metadata files + for path in metadata_paths: + with smart_open.open(path, "wt") as f: + f.write(datetime.now().isoformat()) @classmethod def increment_progressbar(cls, queue: QueueType, /, **kwargs: int) -> Dict[str, int]: @@ -247,126 +392,7 @@ def increment_progressbar(self, queue, /, files = 0, documents = 0): # we use queue.put(tuple(kwargs.get(k, 0) for k in kwargs)) return kwargs - @classmethod - def _run_threaded_progressbar( - cls, - queue: QueueType, - timeout: float, - ): - """Run a progress bar in a separate thread. - - Args: - queue (QueueType): The queue to increment the progress bars. - timeout (float): How often to update the progress bars in seconds. - """ - - sample_queue_output = cls.increment_progressbar(queue) - - with ExitStack() as stack: - pbars = [ - stack.enter_context( - tqdm.tqdm(desc=str(k), unit=str(k)[:1], position=i, unit_scale=True) # pyright: ignore - ) - for i, k in enumerate(sample_queue_output) - ] - - while True: - item = queue.get() - if item is None: - break - - for pbar, value in zip(pbars, item): - pbar.update(value) - - time.sleep(timeout) - - def _debug_run_all( - self, - all_source_paths: List[str], - all_destination_paths: List[str], - all_metadata_paths: List[str], - all_process_kwargs: Union[List[KwargsType], None] = None, - **process_single_kwargs: Any, - ): - """Run files one by one on the main process - - Args: - all_source_paths (List[MultiPath]): The list of source paths to process. - all_destination_paths (List[MultiPath]): The list of destination paths to save. - all_metadata_paths (List[MultiPath]): The locations where to save metadata. - all_process_kwargs (Union[List[KwargsType], None]): Additional kwargs to pass to the process_single - """ - - arguments_iterator = zip( - # source paths - all_source_paths, - # destination paths - all_destination_paths, - # this is where we save the metadata to keep track of which files have been processed - all_metadata_paths, - # additional kwargs to pass to the process_single; if not provided, we use an empty dict - # will be merged with the process_single_kwargs - all_process_kwargs or [{} for _ in all_source_paths], - ) - pbar_queue: QueueType = Queue() - thread = Thread(target=self._run_threaded_progressbar, args=(pbar_queue, self.pbar_timeout), daemon=True) - thread.start() - - for source_path, destination_path, metadata_path, process_kwargs in arguments_iterator: - self._process_single_and_save_status( - source_path=source_path, - destination_path=destination_path, - metadata_path=metadata_path, - queue=pbar_queue, - serialized_kwargs=pickle.dumps({**process_kwargs, **process_single_kwargs}), - ) - - pbar_queue.put(None) - thread.join() - - def __add__(self: BPP, other: BPP) -> BPP: - """Combine two parallel processors into one.""" - if not type(self) is type(other): - raise TypeError(f"Cannot add {type(self)} and {type(other)}") - - # we try combining the two list of include paths; if they are both None, then set the combo back to none - include_paths: Union[List[str], None] = [*(self.include_paths or []), *(other.include_paths or [])] - include_paths = sorted(set(include_paths or [])) if len(include_paths or []) else None - - # do the same for exclude paths - exclude_paths: Union[List[str], None] = [*(self.exclude_paths or []), *(other.exclude_paths or [])] - exclude_paths = sorted(set(exclude_paths or [])) if len(exclude_paths or []) else None - - # for the regex, do a simple or if both are set - regex_pattern: Union[str, None] = None - if self.files_regex_pattern and other.files_regex_pattern: - regex_pattern = "(" + self.files_regex_pattern.pattern + "|" + other.files_regex_pattern.pattern + ")" - elif self.files_regex_pattern: - regex_pattern = self.files_regex_pattern.pattern - elif other.files_regex_pattern: - regex_pattern = other.files_regex_pattern.pattern - - return type(self)( - source_prefix=[*self.src_prefixes, *other.src_prefixes], - destination_prefix=[*self.dst_prefixes, *other.dst_prefixes], - metadata_prefix=[*self.meta_prefixes, *other.meta_prefixes], - num_processes=max(self.num_processes, other.num_processes), - debug=self.debug or other.debug, - seed=self.seed, - pbar_timeout=max(self.pbar_timeout, other.pbar_timeout), - ignore_existing=self.ignore_existing or other.ignore_existing, - include_paths=include_paths, - exclude_paths=exclude_paths, - files_regex_pattern=regex_pattern, - retries_on_error=max(self.retries_on_error, other.retries_on_error), - process_single_kwargs=[*self.process_single_kwargs, *other.process_single_kwargs], - ) - - def __radd__(self: BPP, other: BPP) -> BPP: - """Combine two parallel processors into one.""" - return other.__add__(self) - - def _multiprocessing_run_all( + def _run_all( self, all_source_paths: List[str], all_destination_paths: List[str], @@ -389,47 +415,54 @@ def _multiprocessing_run_all( all_process_kwargs = all_process_kwargs or [{} for _ in all_source_paths] - arguments_iterator = zip( - # source paths - all_source_paths, - # destination paths - all_destination_paths, - # this is where we save the metadata to keep track of which files have been processed - all_metadata_paths, - # additional kwargs to pass to the process_single; if not provided, we use an empty dict - # will be merged with the process_single_kwargs - all_process_kwargs, + batches = list( + batch_iterator( + # source paths + all_source_paths, + # destination paths + all_destination_paths, + # this is where we save the metadata to keep track of which files have been processed + all_metadata_paths, + # additional kwargs to pass to the process_single; if not provided, we use an empty dict + # will be merged with the process_single_kwargs + all_process_kwargs, + # batch size is equal to 1 by default + batch_size=self.batch_size, + ) ) + self.logger.info("Processing in %s batches", len(batches)) - # no need to be wasteful with processes: we only need as many cores a the minimum of the number of - # source paths, destination paths, metadata paths, and process kwargs. - num_processes = min( - self.num_processes, - len(all_source_paths), - len(all_destination_paths), - len(all_metadata_paths), - len(all_process_kwargs), - ) + # no need to be wasteful with processes: we only need as many cores a the number of batches + num_processes = min(self.num_processes, len(batches)) + self.logger.info("Using %s processes", num_processes) - with multiprocessing.Pool(processes=num_processes) as pool: - pbar_queue: QueueType = (manager := multiprocessing.Manager()).Queue() - thread = Thread( - target=self._run_threaded_progressbar, args=(pbar_queue, self.pbar_timeout), daemon=True + with PoolWithDebug(processes=num_processes, debug=self.debug) as pool: + pbar_queue: QueueType = (manager := get_manager(pool)).Queue() + pbar = self.PROGRESS_BAR_CLS( + queue=pbar_queue, min_time=self.pbar_timeout, server=self.progress_bar_mode ) - thread.start() + pbar.start() process_single_fn = partial(self.process_single, queue=pbar_queue) results = [] - for source_path, destination_path, metadata_path, process_kwargs in arguments_iterator: + for source_paths, destination_paths, metadata_paths, process_kwargs in batches: + # we need to merge the process_single_kwargs with the additional kwargs + # mypy is confused by the type of process_kwargs; we need to ignore the error + serialized_kwargs = [ + pickle.dumps({**kw, **process_single_kwargs}) for kw in process_kwargs # type: ignore + ] + process_single_fn = partial( - self._process_single_and_save_status, + self._process_batch_and_save_status, queue=pbar_queue, - source_path=source_path, - destination_path=destination_path, - metadata_path=metadata_path, - # we need to merge the process_single_kwargs with the additional kwargs - serialized_kwargs=pickle.dumps({**process_kwargs, **process_single_kwargs}), + source_paths=source_paths, # pyright: ignore + destination_paths=destination_paths, # pyright: ignore + metadata_paths=metadata_paths, # pyright: ignore + serialized_kwargs=serialized_kwargs, + backoff_max_time=self.backoff_max_time, + backoff_max_tries=self.backoff_max_tries, + backoff_exceptions=self.backoff_exceptions, ) result = pool.apply_async(process_single_fn) results.append(result) @@ -439,9 +472,7 @@ def _multiprocessing_run_all( pool.close() pool.join() - - pbar_queue.put(None) - thread.join() + pbar.stop() manager.shutdown() def _valid_path(self, path: str) -> bool: @@ -453,14 +484,14 @@ def _valid_path(self, path: str) -> bool: return False return True - def _get_all_paths(self) -> AllPathsTuple: + def _get_all_paths(self) -> Tuple[AllPathsTuple, bool]: """Get all paths to process using prefixes provided""" - all_paths = AllPathsTuple.empty() + all_paths = AllPathsTuple.new() for src_prefix, dst_prefix, meta_prefix, kwargs_prefix in zip( self.src_prefixes, self.dst_prefixes, self.meta_prefixes, self.process_single_kwargs ): - current_source_prefixes = sorted(glob_path(src_prefix)) + current_source_prefixes = sorted([src_prefix] if self.skip_source_glob else glob_path(src_prefix)) if len(current_source_prefixes) > 1: # make relative only makes sense if there is more than one path; otherwise, it's unclear @@ -474,45 +505,49 @@ def _get_all_paths(self) -> AllPathsTuple: else: raise ValueError(f"Could not find any files matching {src_prefix}") - # shuffle the order of the files so time estimation in progress bars is more accurate - random.shuffle(rel_paths) + if self.shuffle_src_paths: + # shuffle the order of the files so time estimation in progress bars is more accurate + random.shuffle(rel_paths) - # get a list of which metadata files already exist - existing_metadata_names = set( - re.sub(rf"{METADATA_SUFFIX}$", "", sub_prefix(path, meta_prefix)) - for path in glob_path(meta_prefix) - ) + # # get a list of which metadata files already exist + some_already_processed = False for path in rel_paths: - if not self.ignore_existing and path in existing_metadata_names: - continue + metadata_path = add_suffix(meta_prefix, path) + METADATA_SUFFIX if not self._valid_path(path): + # invalid path; skip + continue + + if not self.ignore_existing and exists(metadata_path): + # metadata file exists, which indicates that the file has already been processed + some_already_processed = True continue # create new paths to pass to taggers all_paths.src.append(add_suffix(prefix, path)) all_paths.dst.append(add_suffix(dst_prefix, path)) - all_paths.meta.append(add_suffix(meta_prefix, path) + METADATA_SUFFIX) + all_paths.meta.append(metadata_path) all_paths.kwargs.append(kwargs_prefix or {}) - return all_paths + return all_paths, some_already_processed def __call__(self, **process_single_kwargs: Any): """Run the processor.""" random.seed(self.seed) - # in case the user wants to override the default kwargs for retries - process_single_kwargs.setdefault("retries_on_error", self.retries_on_error) - - all_paths = self._get_all_paths() + all_paths, some_already_processed = self._get_all_paths() + self.logger.info("Found %s files to process", len(all_paths.src)) - print(f"Found {len(all_paths.src):,} files to process") - - fn = self._debug_run_all if self.debug else self._multiprocessing_run_all + if all_paths.empty: + if some_already_processed: + self.logger.info("All files already processed; skipping.") + return + else: + raise DolmaError("No files found to process.") - fn( + self._run_all( all_source_paths=all_paths.src, all_destination_paths=all_paths.dst, all_metadata_paths=all_paths.meta, diff --git a/python/dolma/core/paths.py b/python/dolma/core/paths.py index ba597e13..46e51545 100644 --- a/python/dolma/core/paths.py +++ b/python/dolma/core/paths.py @@ -1,7 +1,7 @@ import glob import os import re -from functools import partial +from functools import partial, reduce from hashlib import sha256 from itertools import chain from pathlib import Path @@ -223,9 +223,14 @@ def glob_path( protocol, parsed_path = _pathify(path) fs = _get_fs(path) - if fs.isdir(path) and autoglob_dirs: + if autoglob_dirs and fs.isdir(path): path = join_path(protocol, _unescape_glob(parsed_path), "*") + if "*" not in str(path): + # nothing to glob + yield str(path) + return + for gl in fs.glob(path): gl = str(gl) @@ -537,3 +542,25 @@ def split_ext(path: str) -> Tuple[str, Tuple[str, ...], str]: extensions.append(ext) return prot, (*parts[:-1], filename), "".join(reversed(extensions)) + + +def get_unified_path(paths: List[str]) -> str: + """Get a unified path for a list of paths.""" + + if len(paths) == 1: + # if there is only one path, we don't need to unify anything + return paths[0] + + # get shared root for all paths; we will put the unified path here + root, relative = make_relative(paths) + + # get the extension from the first path; assume all paths have the same extension + _, _, ext = split_ext(relative[0]) + + # hash all the sorted relative paths in order to get a unique name + # the type: ignore is needed because mypy fails to infer the type of the lambda + # (the "or" ensures that the lambda returns the same type as the first argument, which is a hash) + h = reduce(lambda h, p: h.update(p.encode()) or h, sorted(relative), sha256()) # type: ignore + + # return the unified path + return join_path(root, h.hexdigest() + ext) diff --git a/python/dolma/core/profile.py b/python/dolma/core/profile.py new file mode 100644 index 00000000..0af8aa9f --- /dev/null +++ b/python/dolma/core/profile.py @@ -0,0 +1,36 @@ +import cProfile +import io +import pstats +from contextlib import ExitStack, contextmanager +from typing import Generator, Optional + +import smart_open + +from .loggers import get_logger + + +@contextmanager +def profiler( + output: Optional[str] = None, sort_key: str = "tottime", lines: int = 100, human_readable: bool = True +) -> Generator[None, None, None]: + logger = get_logger("profiler", "info") + + profile = cProfile.Profile() + logger.info("Starting profiler...") + profile.enable() + yield + profile.disable() + logger.info("Profiler stopped.") + + if not human_readable and output is not None: + logger.info("Dumping profiler stats in binary format to %s...", output) + profile.dump_stats(output) + return + + with ExitStack() as stack: + logger.info("Printing profiler stats %s...", f"to {output}" if output is not None else "to stdout") + output_stream = io.StringIO() if output is None else stack.enter_context(smart_open.open(output, "w")) + ps = pstats.Stats(profile, stream=output_stream).sort_stats(sort_key) + ps.print_stats(lines) + + logger.info("Done printing profiler stats.") diff --git a/python/dolma/core/progressbar.py b/python/dolma/core/progressbar.py new file mode 100644 index 00000000..283d4b01 --- /dev/null +++ b/python/dolma/core/progressbar.py @@ -0,0 +1,330 @@ +import multiprocessing +import time +import warnings +from contextlib import ExitStack +from enum import Enum +from functools import reduce +from hashlib import sha1 +from inspect import Parameter +from inspect import signature as get_signature +from queue import Queue +from threading import Thread +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type + +import tqdm +from rich.progress import filesize +from typing_extensions import TypeAlias, Union + +from .loggers import get_logger + +if TYPE_CHECKING: + from .parallel import BaseParallelProcessor + + +QueueType: TypeAlias = "Queue[Union[None, Tuple[int, ...]]]" + + +class ServerType(Enum): + tqdm = "tqdm" + logger = "logger" + null = "null" + + +class BaseProgressBar: + """One or more progress bars that track progress of a process. + + This class is meant to be subclassed. The subclass must provide one or more attributes of type int, e.g. + + ```python + class MyProgressBar(BaseProgressBar): + files: int = 0 + documents: int = 0 + ``` + + This class can be used for both adding and running through the progress bars. To start: + + ```python + queue = Queue() + pb = MyProgressBar(queue) + pb.start() + + ... # do some work + + pb.stop() + ``` + + it can also be used in a multiprocessing context: + + ```python + with Pool(processes=4) as pool: + queue = mutliprocessing.Manager().Queue() + pb = MyProgressBar(queue) + pb.start() + + ... # do some work + + pool.close() + pool.join() + pb.stop() + ``` + + If you want to use this class to update a queue: + + ```python + pb = MyProgressBar(queue) + pb.files += 1 + pb.documents += 100 + ``` + """ + + def __init__( + self, + queue: QueueType, + min_step: int = 1, + min_time: float = 1e-3, + server: Union[ServerType, str] = "null", + ): + """ + Initialize the ProgressBar object. + + Args: + queue (QueueType): The queue object to track progress. + min_step (int, optional): The minimum step size for progress updates. Defaults to 1. + min_time (float, optional): The minimum time interval between progress updates. Defaults to 1e-1. + thread (bool, optional): Whether to start the progress bar or use object as client. Defaults to False. + """ + self._logger = get_logger(self.__class__.__name__, "warn") + self._queue = queue + self._last_update_time = time.time() + self._last_update_step = 0 + + self._update_every_seconds = min_time + self._update_every_steps = min_step + + for field in self.fields(): + setattr(self, field, 0) + + server_mode = ServerType[server] if isinstance(server, str) else server + if server_mode == ServerType.tqdm: + self._thread: Optional[Thread] = Thread( + target=self._run_tqdm, + kwargs={"queue": queue, "update_every_seconds": min_time, "fields": self.fields()}, + daemon=True, + ) + elif server_mode == ServerType.logger: + self._thread = Thread( + target=self._run_logger, + kwargs={"queue": queue, "update_every_seconds": min_time, "fields": self.fields()}, + daemon=True, + ) + else: + self._thread = None + + def __repr__(self) -> str: + return ( + f"{self.__class__.__name__}(" + f"{', '.join(f'{k}={getattr(self, k)}' for k in self.fields())};" + f" min_step={self._update_every_steps}, min_time={self._update_every_seconds})" + ")" + ) + + def __str__(self) -> str: + return self.__repr__() + + def __setattr__(self, name: str, value: Any) -> None: + super().__setattr__(name, value) + if name in self.fields() and value > 0: + self.update() + + @classmethod + def from_increment_function(cls, processor: "BaseParallelProcessor") -> "Type[BaseProgressBar]": + # print deprecation warning + msg = ( + "Deriving progress bar from `increment_progressbar` is deprecated; add a `PROGRESS_BAR_CLS` " + f"attribute to {type(processor).__name__} instead." + ) + warnings.warn(msg, category=DeprecationWarning, stacklevel=2) + + # checking that the increment_progressbar method is subclassed correctly + sig = get_signature(processor.increment_progressbar) + if "queue" not in sig.parameters or sig.parameters["queue"].kind != Parameter.POSITIONAL_ONLY: + raise AttributeError( + "increment_progressbar must have a positional-only argument named 'queue'; " + "Check that you have subclassed BaseParallelProcessor correctly!" + ) + if "kwargs" in sig.parameters and sig.parameters["kwargs"].kind == Parameter.VAR_KEYWORD: + raise AttributeError( + "increment_progressbar must not have a **kwargs argument; " + "Check that you have subclassed BaseParallelProcessor correctly!" + ) + if any(p.name != "queue" and p.default != 0 for p in sig.parameters.values()): + raise AttributeError( + "increment_progressbar must have a default value of 0 for all arguments except 'queue'; " + "Check that you have subclassed BaseParallelProcessor correctly!" + ) + params = [k for k, p in sig.parameters.items() if k != "queue" and p.kind != Parameter.empty] + h = reduce(lambda h, e: h.update(e.encode()) or h, params, sha1()).hexdigest() # type: ignore + + # create a new class + cls_dict = {"__annotations__": {k: int for k in params}, **{p: 0 for p in params}} + new_cls = type(f"{cls.__name__}{h[-6:]}", (cls,), cls_dict) + return new_cls + + @classmethod + def fields(cls) -> Tuple[str, ...]: + """ + Returns a tuple of field names in the class that are of type int. + + Raises: + ValueError: If the class does not have at least one field of type int. + + Returns: + Tuple[str, ...]: A tuple of field names. + """ + fields: Optional[Tuple[str, ...]] = cls.__dict__.get("__fields__") + + if fields is None: + fields = tuple(n for n, t in getattr(cls, "__annotations__", {}).items() if issubclass(t, int)) + setattr(cls, "__fields__", fields) + + if len(fields) == 0: + raise ValueError(f"Class {cls.__name__} must have at least one field of type int.") + + return fields + + @classmethod + def parse(cls, values: Optional[Tuple[int, ...]]) -> Dict[str, int]: + """ + Parses the value from the queue and returns a dictionary mapping field names to their corresponding values. + + Args: + values (Optional[Tuple[int, ...]]): The values to be parsed for the queue. + + Returns: + Dict[str, int]: A dictionary mapping field names to their corresponding values. + """ + if not values: + return {k: 0 for k in cls.fields()} + return {k: v for k, v in zip(cls.fields(), values)} + + def _update(self): + # get the current values + update = tuple(getattr(self, k, 0) for k in self.fields()) + + # time to do an update + self._queue.put_nowait(update) + + # reset the steps + self._last_update_step = 0 + self._last_update_time = time.time() + + # reset the steps + for k in self.fields(): + setattr(self, k, 0) + + def update(self): + # update the number of steps since the last update + self._last_update_step += 1 + + if self._update_every_steps > self._last_update_step: + return + + time_before_update = self._last_update_time + self._update() + + # check if we wanna update frequency based on steps + if self._queue.qsize() >= multiprocessing.cpu_count(): + self._update_every_steps *= 2 + return + + # check if we wanna update frequency based on time + if (self._last_update_time - time_before_update) < self._update_every_seconds: + self._update_every_steps *= 2 + return + + @staticmethod + def _run_tqdm(queue: QueueType, update_every_seconds: float, fields: Tuple[str, ...]): + """ + Runs the progress bar. + + This method initializes and updates the progress bars based on the items in the queue. + It continuously retrieves items from the queue and updates the progress bars accordingly. + The method exits when a `None` item is retrieved from the queue. + + Returns: + None + """ + with ExitStack() as stack: + pbars = [ + stack.enter_context(tqdm.tqdm(desc=k, unit=k[:1], position=i, unit_scale=True)) # pyright: ignore + for i, k in enumerate(fields) + ] + + while True: + # loop until we get a None + item = queue.get() + if item is None: + break + + for pbar, value in zip(pbars, item): + pbar.update(value) + + time.sleep(update_every_seconds) + + @staticmethod + def _run_logger(queue: QueueType, update_every_seconds: float, fields: Tuple[str, ...]): + """ + Run the progress bar update loop. + + Args: + queue (QueueType): The queue to retrieve items from. + update_every_seconds (float): The interval between each update in seconds. + fields (Tuple[str, ...]): The fields to track and display in the progress bar. + + Returns: + None + """ + total_counters = {k: 0 for k in fields} + logger = get_logger("progress", "info") + + while True: + # loop until we get a None + item = queue.get() + if item is None: + break + + messages = [] + for k, v in zip(fields, item): + total_counters[k] += v + unit, suffix = filesize.pick_unit_and_suffix( + total_counters[k], ["", "K", "M", "G", "T", "P", "E", "Z", "Y"], 1000 + ) + precision = 1 if suffix else 0 + messages.append(f"{k}: {total_counters[k] / unit:,.{precision}f}{suffix} (+{v:,})") + + logger.info(", ".join(messages)) + time.sleep(update_every_seconds) + + def start(self): + """Run the progress bar in a separate thread.""" + if self._thread: + self._thread.start() + + def stop(self): + """Stop the progress bar. + + This method stops the progress bar by adding a `None` item to the queue and joining the thread. + """ + self._update() + + if self._thread is not None: + self._queue.put(None) + time.sleep(self._update_every_seconds * 2) + self._thread.join() + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + self.stop() diff --git a/python/dolma/core/registry.py b/python/dolma/core/registry.py index cc132627..cfa6a4d7 100644 --- a/python/dolma/core/registry.py +++ b/python/dolma/core/registry.py @@ -1,18 +1,30 @@ -from typing import Callable, Dict, Generator, Generic, Tuple, Type, TypeVar +from typing import Callable, Dict, Generator, Generic, Optional, Tuple, Type, TypeVar from .taggers import BaseTagger -T = TypeVar("T", bound=Type) -R = TypeVar("R", bound=Type) +T = TypeVar("T") +R = TypeVar("R") class BaseRegistry(Generic[T]): """A registry for objects.""" - _registry_storage: Dict[str, T] + _registry_of_registries: Dict[str, Type["BaseRegistry"]] = {} + _registry_storage: Dict[str, Tuple[T, Optional[str]]] @classmethod - def _get_storage(cls) -> Dict[str, T]: + def _add_to_registry_of_registries(cls) -> None: + name = cls.__name__ + if name not in cls._registry_of_registries: + cls._registry_of_registries[name] = cls + + @classmethod + def registries(cls) -> Generator[Tuple[str, Type["BaseRegistry"]], None, None]: + """Yield all registries in the registry of registries.""" + yield from sorted(cls._registry_of_registries.items()) + + @classmethod + def _get_storage(cls) -> Dict[str, Tuple[T, Optional[str]]]: if not hasattr(cls, "_registry_storage"): cls._registry_storage = {} return cls._registry_storage # pyright: ignore @@ -20,23 +32,36 @@ def _get_storage(cls) -> Dict[str, T]: @classmethod def items(cls) -> Generator[Tuple[str, T], None, None]: """Yield all items in the registry.""" - yield from sorted(cls._get_storage().items()) + yield from sorted((n, t) for (n, (t, _)) in cls._get_storage().items()) @classmethod - def add(cls, name: str) -> Callable[[R], R]: + def items_with_description(cls) -> Generator[Tuple[str, T, Optional[str]], None, None]: + """Yield all items in the registry with their descriptions.""" + yield from sorted((n, t, d) for (n, (t, d)) in cls._get_storage().items()) + + @classmethod + def add(cls, name: str, desc: Optional[str] = None) -> Callable[[T], T]: """Add a class to the registry.""" - def _add(tagger_self: T, tagger_name: str = name, cls_: Type[BaseRegistry] = cls) -> T: + # Add the registry to the registry of registries + cls._add_to_registry_of_registries() + + def _add( + tagger_self: T, + tagger_name: str = name, + tagger_desc: Optional[str] = desc, + tagger_cls: Type[BaseRegistry] = cls, + ) -> T: """Add a tagger to the registry using tagger_name as the name.""" - if tagger_name in cls_._get_storage() and cls_._get_storage()[tagger_name] != tagger_self: + if tagger_name in tagger_cls._get_storage() and tagger_cls.get(tagger_name) != tagger_self: if tagger_self.__module__ == "__main__": return tagger_self raise ValueError(f"Tagger {tagger_name} already exists") - cls_._get_storage()[tagger_name] = tagger_self + tagger_cls._get_storage()[tagger_name] = (tagger_self, tagger_desc) return tagger_self - return _add # type: ignore + return _add @classmethod def remove(cls, name: str) -> bool: @@ -57,7 +82,8 @@ def get(cls, name: str) -> T: if name not in cls._get_storage(): tagger_names = ", ".join([tn for tn, _ in cls.items()]) raise ValueError(f"Unknown tagger {name}; available taggers: {tagger_names}") - return cls._get_storage()[name] + t, _ = cls._get_storage()[name] + return t class TaggerRegistry(BaseRegistry[Type[BaseTagger]]): diff --git a/python/dolma/core/runtime.py b/python/dolma/core/runtime.py index ac5e2a23..ff460fa9 100644 --- a/python/dolma/core/runtime.py +++ b/python/dolma/core/runtime.py @@ -1,5 +1,4 @@ -import io -import multiprocessing +# import multiprocessing import tempfile from contextlib import ExitStack, contextmanager from typing import ( @@ -18,7 +17,7 @@ import msgspec import smart_open -from dolma.core.taggers import BaseTaggerWithMetadata +from dolma.core.progressbar import BaseProgressBar from .data_types import ( InputSpec, @@ -29,7 +28,9 @@ from .errors import DolmaFatalError, DolmaRetryableFailure, DolmaShardError from .parallel import BaseParallelProcessor, QueueType from .paths import delete_dir, join_path, make_relative, mkdir_p, split_glob, split_path +from .profile import profiler from .registry import TaggerRegistry +from .taggers import BaseTaggerWithMetadata from .utils import import_modules, make_variable_name # this placeholder gets used when a user has provided no experiment name, and we want to use taggers' @@ -221,20 +222,13 @@ def _write_sample_to_streams( output_streams[stream_path].write(output) -class TaggerProcessor(BaseParallelProcessor): - @classmethod - def increment_progressbar( # type: ignore - cls, - queue: QueueType, # queue must be the first argument, and it should be a positional-only argument - /, - files: int = 0, - documents: int = 0, - ) -> Dict[str, int]: - """We override this method to specify which units we want to keep track of in a progress bar. - Specifically, we keep track of files and documents in this example. Their default value must be zero.""" +class TaggerProcessorProgessBar(BaseProgressBar): + files: int = 0 + documents: int = 0 - # we call the super method to increment the progress bar - return super().increment_progressbar(queue, files=files, documents=documents) + +class TaggerProcessor(BaseParallelProcessor): + PROGRESS_BAR_CLS = TaggerProcessorProgessBar @classmethod def process_single( @@ -273,12 +267,9 @@ def process_single( # maximum numbers of lines to process steps: Union[int, None] = kwargs.get("steps", None) - # interval at which to update the progress bar; will double if it gets - # too full - update_interval = 1 - - # running document count; gets reset every time we update the progress bar - docs_cnt = 0 + # compression configuration + compression_input = kwargs.get("compression_input", None) + compression_output = kwargs.get("compression_output", None) # total number of documents processed total_docs_cnt = 0 @@ -292,10 +283,15 @@ def process_single( decoder = msgspec.json.Decoder(InputSpec) with ExitStack() as stack: - in_stream = stack.enter_context(smart_open.open(source_path, "rt", encoding="utf-8")) + in_stream = stack.enter_context( + smart_open.open(source_path, "rt", encoding="utf-8", compression=compression_input) + ) output_streams = stack.enter_context( - _make_output_streams(taggers_paths=taggers_paths, mode="wt", encoding="utf-8") + _make_output_streams( + taggers_paths=taggers_paths, mode="wt", encoding="utf-8", compression=compression_output + ) ) + pbar = stack.enter_context(TaggerProcessorProgessBar(queue)) try: for raw in in_stream: row = decoder.decode(raw) @@ -310,23 +306,13 @@ def process_single( samples_collectors[tagger_name] = tagger.tag(row) # increment the number of documents processed so far - docs_cnt += 1 + pbar.documents += 1 total_docs_cnt += 1 if steps is not None and total_docs_cnt >= steps: # if we have reached the maximum number of steps, we break break - if docs_cnt % update_interval == 0: - # update the progress bar every 1000 documents to prevent - # buffering - cls.increment_progressbar(queue, documents=docs_cnt) - docs_cnt = 0 - - if queue.qsize() >= multiprocessing.cpu_count(): - # double the update interval if the queue is full - update_interval *= 2 - except Exception as exp: # handle any exception that might have occurred msg = f"Failed to process {source_path} due to {exp.__class__.__name__}: {' '.join(exp.args)}" @@ -339,28 +325,8 @@ def process_single( else: raise DolmaFatalError(msg) from exp - # increment the files progress bar - cls.increment_progressbar(queue, files=1, documents=docs_cnt) - - -@contextmanager -def profiler( - output: Optional[str] = None, - sort_key: str = "tottime", - lines: int = 100, -) -> Generator[None, None, None]: - import cProfile - import pstats - - profile = cProfile.Profile() - profile.enable() - yield - profile.disable() - - with ExitStack() as stack: - output_stream = io.StringIO() if output is None else stack.enter_context(smart_open.open(output, "w")) - ps = pstats.Stats(profile, stream=output_stream).sort_stats(sort_key) - ps.print_stats(lines) + # increment the files progress bar + pbar.files += 1 @contextmanager @@ -392,6 +358,8 @@ def create_and_run_tagger( profile_steps: Optional[int] = None, profile_sort_key: str = "tottime", profile_lines: int = 100, + compression_input: Optional[str] = None, + compression_output: Optional[str] = None, ): """This function creates a tagger and runs it on a list of documents. @@ -423,8 +391,26 @@ def create_and_run_tagger( profile_steps (Optional[int], optional): Number of steps to profile; if not provided, all steps will be profiled. Defaults to None. profile_sort_key (str, optional): Sort key for the profiling output. Defaults to 'tottime'. + compression_input (Optional[str], optional): Compression algorithm to use for input files. If None, + compression will be inferred from the input file extension. Defaults to None. + compression_output (Optional[str], optional): Compression algorithm to use for output files. If None, + compression will be inferred from the output file extension. Defaults to None. """ + # get a list of supported compression types + compression_type = smart_open.compression.get_supported_compression_types() + + # if compression is not provided, set it to "infer_from_extension"; this is to maintain consistency with + # how compression is specified in the mixer/deduper code + compression_input = compression_input or "infer_from_extension" + compression_output = compression_output or "infer_from_extension" + + # check if compression is supported + if compression_input not in compression_type: + raise ValueError(f"Compression {compression_input} is not supported") + if compression_output not in compression_type: + raise ValueError(f"Compression {compression_output} is not supported") + # before pre-caching taggers, import any taggers modules if taggers_modules is not None: import_modules(taggers_modules) @@ -467,7 +453,7 @@ def create_and_run_tagger( debug=debug or profile_enable, # if profile is true, debug must be true seed=seed, ignore_existing=ignore_existing, - retries_on_error=retries_on_error, + backoff_max_tries=retries_on_error, num_processes=num_processes, ) @@ -486,4 +472,6 @@ def create_and_run_tagger( taggers_modules=taggers_modules, skip_on_failure=skip_on_failure, steps=profile_steps, + compression_input=compression_input, + compression_output=compression_output, ) diff --git a/python/dolma/core/taggers.py b/python/dolma/core/taggers.py index 59a414f7..1ed99726 100644 --- a/python/dolma/core/taggers.py +++ b/python/dolma/core/taggers.py @@ -17,9 +17,7 @@ InputSpecWithMetadata, TaggerOutputDictType, ) - -# digits after the decimal point -TAGGER_SCORE_PRECISION = 5 +from .utils import format_span_output class BaseTagger: @@ -46,8 +44,7 @@ def predict(self, doc: Document) -> DocResult: def group_output(self, doc_result: DocResult) -> TaggerOutputDictType: tagger_output: TaggerOutputDictType = {field: [] for field in self.defaults} for span in doc_result.spans: - output = (span.start, span.end, round(float(span.score), TAGGER_SCORE_PRECISION)) - tagger_output.setdefault(span.type, []).append(output) + tagger_output.setdefault(span.type, []).append(format_span_output(span)) return tagger_output def tag(self, row: InputSpec) -> TaggerOutputDictType: diff --git a/python/dolma/core/utils.py b/python/dolma/core/utils.py index c8149e74..6c4413d6 100644 --- a/python/dolma/core/utils.py +++ b/python/dolma/core/utils.py @@ -4,14 +4,8 @@ import re import string import sys -from typing import List, Union, cast - -try: - import blingfire - - BLINGFIRE_AVAILABLE = True -except Exception: - BLINGFIRE_AVAILABLE = False +from itertools import islice +from typing import Generator, Iterable, List, Tuple, TypeVar, Union, cast import nltk import uniseg.wordbreak @@ -26,13 +20,26 @@ except LookupError: nltk.download("punkt") - -from .data_types import TextSlice +from .data_types import Span, TextSlice from .loggers import get_logger +try: + import blingfire + + BLINGFIRE_AVAILABLE = True +except (ImportError, OSError): + BLINGFIRE_AVAILABLE = False + + sent_tokenizer = PunktSentenceTokenizer() logger = get_logger(__name__) +T = TypeVar("T") + + +# digits after the decimal point +TAGGER_SCORE_PRECISION = 5 + def make_variable_name(name: str, remove_multiple_underscores: bool = False) -> str: # use underscores for any non-valid characters in variable name @@ -48,6 +55,16 @@ def make_variable_name(name: str, remove_multiple_underscores: bool = False) -> return name +def format_span_output(span: Span) -> Tuple[int, int, float]: + """Formats a span for output.""" + return (span.start, span.end, round(float(span.score), TAGGER_SCORE_PRECISION)) + + +def format_span_key(experiment: str, tagger: str, span: Span) -> str: + """Formats a span key for output.""" + return f"{experiment}__{tagger}__{make_variable_name(span.type)}" + + def split_words(text: str, remove_empty: bool = True) -> List[TextSlice]: """ Split a string into words, as defined by the unicode standard. @@ -138,7 +155,7 @@ def import_modules(modules_path: Union[List[str], None]): sys.path.insert(0, module_parent) importlib.import_module(module_name) elif module_path in sys.modules[module_name].__path__: - logger.info(f"{module_path} has already been imported.") + logger.info("%s has already been imported.", module_path) else: raise ImportError( f"Failed to import {module_path} because the corresponding module name " @@ -154,6 +171,27 @@ def dataclass_to_dict(dataclass_instance) -> dict: return cast(dict, om.to_object(om.structured(dataclass_instance))) +def batch_iterator( + *iterables: Iterable[T], batch_size: int = 1, drop_last: bool = False +) -> Generator[List[Tuple[T, ...]], None, None]: + """ + Group one or more iterables into batches of size `batch_size`. + + Args: + iterables (Iterable[T]): One or more iterables to group into batches. + batch_size (int): The size of each batch. Defaults to 1. + drop_last (bool): Whether to drop the last batch if it is smaller than `batch_size`. Defaults to False. + """ + grouped_iterator = iter(zip(*iterables)) + while True: + batch = list(islice(grouped_iterator, batch_size)) + if not batch: + break + if len(batch) < batch_size and drop_last: + break + yield list(zip(*batch)) + + def add_compression(): """ Adds support for zstandard (.zst) compression format to the smart_open library. diff --git a/python/dolma/taggers/__init__.py b/python/dolma/taggers/__init__.py index ece1ae3b..08ec6bec 100644 --- a/python/dolma/taggers/__init__.py +++ b/python/dolma/taggers/__init__.py @@ -11,6 +11,7 @@ quality, repetitions, sampling, + science, tokenizers, url, ) diff --git a/python/dolma/taggers/language.py b/python/dolma/taggers/language.py index 66cb4c6d..6feae86d 100644 --- a/python/dolma/taggers/language.py +++ b/python/dolma/taggers/language.py @@ -182,6 +182,17 @@ def predict_text(self, text: str) -> List[Tuple[str, float]]: return filtered_preds # pyright: ignore +@TaggerRegistry.add("ft_dolma_doc_eng") +class FastTextEnglishDolmaTagger(FastTextEnglishLanguageDocumentTagger): + INCLUDE_NEGATIVE = False + PREDICT_ON_PARAGRAPHS = False + + def predict_text(self, text: str) -> List[Tuple[str, float]]: + preds = super().predict_text(text) + filtered_preds = [(lang, score) for lang, score in preds if lang == "en" and score > 0.5] + return filtered_preds # pyright: ignore + + @TaggerRegistry.add("ft_lang_id_en_only_v2") class FastTextEnglishOnlyLanguageDocumentTagger(FastTextEnglishLanguageDocumentTagger): INCLUDE_NEGATIVE = False diff --git a/python/dolma/taggers/licenses.py b/python/dolma/taggers/licenses.py index 76ca572b..dbb7d63a 100644 --- a/python/dolma/taggers/licenses.py +++ b/python/dolma/taggers/licenses.py @@ -6,63 +6,98 @@ """ -from typing import List, Optional +import re +from typing import TYPE_CHECKING, Any, List, Optional, Tuple import regex +from necessary import necessary from ..core.data_types import DocResult, DocumentWithMetadata, Span from ..core.registry import TaggerRegistry from ..core.taggers import BaseTaggerWithMetadata +with necessary("hyperscan", soft=True) as HYPERSCAN_AVAILABLE: + if TYPE_CHECKING or HYPERSCAN_AVAILABLE: + from hyperscan import Database + @TaggerRegistry.add("cc_re") class CreativeCommonsRegexLicenseExtractor(BaseTaggerWithMetadata): """Adapted from https://github.com/dkpro/dkpro-c4corpus/blob/da61281a8a77fad0d6a7d27c06b5e2fe3282e28f/dkpro-c4corpus-license/src/main/java/de/tudarmstadt/ukp/dkpro/c4corpus/license/impl/LicenseDetectorBasic.java""" # noqa - PRE_REGEX_SEARCH = ("creativecommons.org/licenses", "creativecommons.org/publicdomain") - LICENSE_TYPE = "by(-nc)?(-nd)?(-sa)?" - VERSION = r"\d+\.\d+" - LANG_PREFIX = r"\w{2}" - RE_LICENSE_ATTRIBUTE_PATTERN = regex.compile( + PRE_REGEX = (rb"creativecommons\.org/licenses", rb"creativecommons\.org/publicdomain") + _LICENSE_TYPE = "by(-nc)?(-nd)?(-sa)?" + _LICENSE_VERSION = r"\d+\.\d+" + _LICENSE_LANG_PREFIX = r"\w{2}" + LICENSE_PATTERN = ( "<(a|A|meta)\\s[\\w\\p{Punct}\\s=]*\n*(href|HREF|content)" "=('|\"|")?http(s*)://creativecommons\\.org/" - f"((licenses/(?P{LICENSE_TYPE}))|(?Ppublicdomain/(zero|certification|mark)))" - f"(?P/{VERSION})?" - f"((/{LANG_PREFIX})?/((deed|legalcode)\\.)?(?P{LANG_PREFIX}))?.*?('|\"|").*?>" + f"((licenses/(?P{_LICENSE_TYPE}))|(?Ppublicdomain/(zero|certification|mark)))" + f"(?P/{_LICENSE_VERSION})?" + f"((/{_LICENSE_LANG_PREFIX})?/((deed|legalcode)\\.)?(?P{_LICENSE_LANG_PREFIX}))?.*?('|\"|").*?>" ) def __init__(self): - self.has_type_group = "type" in self.RE_LICENSE_ATTRIBUTE_PATTERN.groupindex - self.has_version_group = "version" in self.RE_LICENSE_ATTRIBUTE_PATTERN.groupindex - self.has_lang_group = "lang" in self.RE_LICENSE_ATTRIBUTE_PATTERN.groupindex + assert HYPERSCAN_AVAILABLE, "Hyperscan is not available; please install with `pip install hyperscan`." + self.db = Database() + self.db.compile( + expressions=self.PRE_REGEX, + ids=list(range(len(self.PRE_REGEX))), + elements=len(self.PRE_REGEX), + flags=[0 for _ in self.PRE_REGEX], + ) + + self.license_matcher = regex.compile(self.LICENSE_PATTERN.encode("utf-8")) + + self.has_type_group = "type" in self.license_matcher.groupindex + self.has_version_group = "version" in self.license_matcher.groupindex + self.has_lang_group = "lang" in self.license_matcher.groupindex if not self.has_type_group: raise ValueError("License regex must have a `type` group") super().__init__() + @staticmethod + def _on_match(id_: int, from_: int, to: int, flags: int, context: Optional[Any] = None) -> None: + if context is not None: + context.append((id_, from_, to, flags)) + def predict(self, doc: DocumentWithMetadata) -> DocResult: # type: ignore - html: Optional[str] = doc.metadata.get("html", None) + html: Optional[bytes] = doc.metadata.get("html", None) if html is None: raise ValueError("Cannot find `html` key in metadata.") - if not any(p in html for p in self.PRE_REGEX_SEARCH): + content: List[Tuple[int, int, int, int]] = [] + self.db.scan(html, match_event_handler=self._on_match, context=content) + if not content: return DocResult(doc=doc, spans=[]) spans: List[Span] = [] - for i, match in enumerate(self.RE_LICENSE_ATTRIBUTE_PATTERN.finditer(html)): - license_string = match.group("type") + for i, match in enumerate(self.license_matcher.finditer(html)): + license_string = match.group("type").decode("utf-8") if self.has_version_group and (version := match.group("version")) is not None: - license_string += f"_{version.strip('/')}" + license_string += f"_{version.strip(b'/').decode('utf-8')}" if self.has_lang_group and (lang := match.group("lang")) is not None: - license_string += f"_{lang}" + license_string += f"_{lang.decode('utf-8')}" + + # get location of match in the document + match_start, match_end = match.span() # if multiple license matches are found, the confidence is lowered # for each match. The first match has a confidence of 1.0, the second # has a confidence of 0.75, the third 0.667, the fourth 0.625, etc. score = 0.5 + 0.5 / (i + 1.0) - spans.append(Span(start=0, end=len(doc.text), type=f"cc_{license_string}", score=score)) + spans.append( + Span( + start=match_start, + end=match_end, + type=f"cc_{license_string}", + score=score, + location="metadata.html", + ) + ) return DocResult(doc=doc, spans=spans) @@ -71,7 +106,84 @@ def predict(self, doc: DocumentWithMetadata) -> DocResult: # type: ignore class CreativeCommonsFastRegexHtmlExtractor(CreativeCommonsRegexLicenseExtractor): """Adapted from https://github.com/dkpro/dkpro-c4corpus/blob/da61281a8a77fad0d6a7d27c06b5e2fe3282e28f/dkpro-c4corpus-license/src/main/java/de/tudarmstadt/ukp/dkpro/c4corpus/license/impl/FastRegexLicenceDetector.java""" # noqa - RE_LICENSE_ATTRIBUTE_PATTERN = regex.compile( + LICENSE_PATTERN = ( "http[s]?://creativecommons\\.org/licenses/" '(?Pby|by-sa|by-nd|by-nc|by-nc-sa|by-nc-nd|publicdomain)["/ >]' ) + + +@TaggerRegistry.add("copyright") +class CopyrightTagger(BaseTaggerWithMetadata): + """Extracts copyright notices from HTML documents.""" + + COPYRIGHT_KEYWORDS = [ + "©", + "Copyright", + "版权", + "Derechos de autor", + "حقوق النشر", + "Direitos autorais", + "著作権", + "Авторское право", + "Urheberrecht", + "Droit d'auteur", + "저작권", + "Diritto d'autore", + "Telif hakkı", + "Bản quyền", + "Prawo autorskie", + "Auteursrecht", + "Hak cipta", + "ลิขสิทธิ์", + "حق نشر", + "प्रतिलिप्यधिकार", + "কপিরাইট", + "All rights reserved", + "保留所有权利", + "Todos los derechos reservados", + "جميع الحقوق محفوظة", + "Todos os direitos reservados", + "無断転載を禁じます", + "Все права защищены", + "Alle Rechte vorbehalten", + "Tous droits réservés", + "판권 소유", + "Tutti i diritti riservati", + "Tüm hakları saklıdır", + "Bảo lưu mọi quyền", + "Wszelkie prawa zastrzeżone", + "Alle rechten voorbehouden", + "Hak cipta dilindungi", + "สงวนลิขสิทธิ์", + "کلیه حقوق محفوظ است", + "सर्वाधिकार सुरक्षित", + "সর্বস্বত্ব সংরক্ষিত", + ] + + def __init__(self): + assert HYPERSCAN_AVAILABLE, "Hyperscan is not available; please install with `pip install hyperscan`." + self.db = Database() + + all_expressions = [re.escape(keyword).encode("utf-8") for keyword in self.COPYRIGHT_KEYWORDS] + all_expressions += [exp.lower() for exp in all_expressions] + self.db.compile( + expressions=all_expressions, + ids=list(range(len(all_expressions))), + elements=len(all_expressions), + flags=[0 for _ in all_expressions], + ) + + @staticmethod + def _on_match(id_: int, from_: int, to: int, flags: int, context: Optional[Any] = None) -> None: + if context is not None: + context.append(Span(start=from_, end=to, type="copyright")) + + def predict(self, doc: DocumentWithMetadata) -> DocResult: # type: ignore + html: Optional[bytes] = doc.metadata.get("html", None) + if html is None: + raise ValueError("Cannot find `html` key in metadata.") + + # extract copyright notices + content: List[Span] = [] + self.db.scan(html, match_event_handler=self._on_match, context=content) + return DocResult(doc=doc, spans=content) diff --git a/python/dolma/taggers/repetitions/__init__.py b/python/dolma/taggers/repetitions/__init__.py index 9f4e08ac..87eefb92 100644 --- a/python/dolma/taggers/repetitions/__init__.py +++ b/python/dolma/taggers/repetitions/__init__.py @@ -4,10 +4,20 @@ TokenizerRepetitionsSkipEmptyTagger, TokenizerRepetitionsTagger, ) +from .top_k import ( + Top5TokensTagger, + Top10TokensTagger, + Top20TokensTagger, + Top50TokensTagger, + Top100TokensTagger, +) __all__ = [ - "RepetitionsTagger", "ParagraphRepetitionsTagger", - "TokenizerRepetitionsTagger", + "RepetitionsTagger", "TokenizerRepetitionsSkipEmptyTagger", + "TokenizerRepetitionsTagger", + "Top5TokensTagger", + "Top10TokensTagger", + "Top20TokensTagger", ] diff --git a/python/dolma/taggers/repetitions/top_k.py b/python/dolma/taggers/repetitions/top_k.py new file mode 100644 index 00000000..a4f799ba --- /dev/null +++ b/python/dolma/taggers/repetitions/top_k.py @@ -0,0 +1,56 @@ +""" +Taggers to detect the top K most repeated sequences in a document. + +@soldni +""" + +import re +from collections import Counter + +from ...core.data_types import DocResult, Document, Span +from ...core.registry import TaggerRegistry +from ...core.taggers import BaseTagger + + +class BaseTopKTagger(BaseTagger): + K: int + EXPRESSION: str = r"(\W+|[A-Z]\w+)" + + def __init__(self): + self.splitter = re.compile(self.EXPRESSION) + + def predict(self, doc: Document) -> DocResult: + """Predict method for the tagger.""" + tokens = [ws for w in self.splitter.split(doc.text) if (ws := w.strip())] + counter = Counter(tokens) + spans = [ + Span(start=(pos := doc.text.find(k)), end=pos + len(k), type=str(i), score=v / len(tokens)) + for i, (k, v) in enumerate(counter.most_common(self.K), start=1) + ] + + return DocResult(doc=doc, spans=spans) + + +@TaggerRegistry.add("top_5_tokens") +class Top5TokensTagger(BaseTopKTagger): + K = 5 + + +@TaggerRegistry.add("top_10_tokens") +class Top10TokensTagger(BaseTopKTagger): + K = 10 + + +@TaggerRegistry.add("top_20_tokens") +class Top20TokensTagger(BaseTopKTagger): + K = 20 + + +@TaggerRegistry.add("top_50_tokens") +class Top50TokensTagger(BaseTopKTagger): + K = 50 + + +@TaggerRegistry.add("top_100_tokens") +class Top100TokensTagger(BaseTopKTagger): + K = 100 diff --git a/python/dolma/taggers/science.py b/python/dolma/taggers/science.py new file mode 100644 index 00000000..8a0919eb --- /dev/null +++ b/python/dolma/taggers/science.py @@ -0,0 +1,561 @@ +import re +from typing import TYPE_CHECKING, Any, Iterable, List, Optional + +from necessary import necessary + +from dolma.core.ft_tagger import Prediction + +with necessary("acora", soft=True) as ACORA_AVAILABLE: + if TYPE_CHECKING or ACORA_AVAILABLE: + from acora import AcoraBuilder # pyright: ignore + + +with necessary("hyperscan", soft=True) as HYPERSCAN_AVAILABLE: + if TYPE_CHECKING or HYPERSCAN_AVAILABLE: + from hyperscan import Database + +from ..core.data_types import DocResult, DocumentWithMetadata, Span, TextSlice +from ..core.ft_tagger import BaseFastTextTagger +from ..core.registry import TaggerRegistry +from ..core.taggers import BaseTaggerWithMetadata + + +class BaseHTMLKeywordLookupTagger(BaseTaggerWithMetadata): + KEYWORDS: List[bytes] + TYPE: str + + def __init__(self): + assert ACORA_AVAILABLE, "Acora is not available; please install with `pip install acora`." + + builder = AcoraBuilder() + builder.update(self.KEYWORDS) + self.acora = builder.build() + + def _get_content(self, doc: DocumentWithMetadata) -> bytes: + html: Optional[bytes] = doc.metadata.get("html", None) + if html is None: + raise ValueError("Cannot find `html` key in metadata.") + return html + + def predict(self, doc: DocumentWithMetadata) -> DocResult: # type: ignore + content = self._get_content(doc) + + # check if there's a match; if yes, return immediately + for kw, pos in self.acora.finditer(content): + return DocResult( + doc=doc, + spans=[Span(start=pos, end=pos + len(kw), type=self.TYPE, score=1, location="metadata.html")], + ) + + # if no match, return empty spans + return DocResult(doc=doc, spans=[]) + + +@TaggerRegistry.add("owm_math_v1") +class OpenWebMathContainsMathTagger(BaseHTMLKeywordLookupTagger): + TYPE = "math" + KEYWORDS = [ + b"MathJax", + b"mathjax", + b" bytes: + html: Optional[bytes] = doc.metadata.get("html", None) + if html is None: + raise ValueError("Cannot find `html` key in metadata.") + return html + + @staticmethod + def _on_match(id_: int, from_: int, to: int, flags: int, context: Optional[Any] = None) -> None: + if context is not None: + context.append((id_, from_, to, flags)) + + def predict(self, doc: DocumentWithMetadata) -> DocResult: # type: ignore + content = self._get_content(doc) + + context: List[tuple] = [] + self.db.scan(content, match_event_handler=self._on_match, context=context) + if context: + start, end = context[0][1], context[0][2] + return DocResult( + doc=doc, spans=[Span(start=start, end=end, type=self.TYPE, score=1, location="metadata.html")] + ) + + # if no match, return empty spans + return DocResult(doc=doc, spans=[]) + + +@TaggerRegistry.add("owm_math_v2") +class HyperscanOpenWebMathContainsMathTagger(HyperscanHTMLKeywordLookupTagger): + TYPE = "math" + KEYWORDS = [ + rb"MathJax", + rb"mathjax", + rb" Iterable[Prediction]: + text = re.sub(r"\s+", " ", text_slice.text).strip() + preds = {label: float(score) for label, score in zip(*self.classifier.predict(text, k=-1))} + if preds["__label__1"] > self.FT_THRESHOLD: + return [Prediction(label="science", score=preds["__label__1"])] + return [] + + +@TaggerRegistry.add("owmV2_FTsciV1_comb") +class OwmMathLatexFtScienceCombined(HyperscanHTMLKeywordLookupTagger, FastTextScienceTagger): + TYPE = "math-latex" + MODEL_PATH = FastTextScienceTagger.MODEL_PATH # pyright: ignore + KEYWORDS = ( + HyperscanOpenWebMathContainsMathTagger.KEYWORDS # pyright: ignore + + HyperscanOpenWebMathContainsLatexTagger.KEYWORDS # pyright: ignore + ) + + def __init__(self): + HyperscanHTMLKeywordLookupTagger.__init__(self) + FastTextScienceTagger.__init__(self) + + def predict(self, doc: DocumentWithMetadata) -> DocResult: # type: ignore + keyword_result = HyperscanHTMLKeywordLookupTagger.predict(self, doc) + + if keyword_result.spans: + return keyword_result + + return FastTextScienceTagger.predict(self, doc) + + +@TaggerRegistry.add("owmV2_FTsciV1_comb_lth") +class OwmMathLatexFtScienceCombinedLowThreshold(OwmMathLatexFtScienceCombined): + FT_THRESHOLD = 0.20 + + +@TaggerRegistry.add("owmV2_FTsciV1_comb_hth") +class OwmMathLatexFtScienceCombinedHighThreshold(OwmMathLatexFtScienceCombined): + FT_THRESHOLD = 0.50 + +@TaggerRegistry.add("owmV2_FTsciV1_comb_vhth") +class OwmMathLatexFtScienceCombinedVeryHighThreshold(OwmMathLatexFtScienceCombined): + FT_THRESHOLD = 0.70 + +@TaggerRegistry.add("owmV2_FTsciV1_comb_vvhth") +class OwmMathLatexFtScienceCombinedVeryVeryHighThreshold(OwmMathLatexFtScienceCombined): + FT_THRESHOLD = 0.90 + +@TaggerRegistry.add("ft_science_v1_qt") +class FastTextScienceTaggerQuantized(FastTextScienceTagger): + MODEL_PATH = "https://dolma-artifacts.org/fasttext_models/scipile/model_exp_20000_0.3_owm_10000_syn_5000_wiki_5000_pretrained_dimreduced_quantized.ftz" # noqa: E501 + + +@TaggerRegistry.add("owmV2_FTsciV1_comb_qt") +class OwmMathLatexFtScienceCombinedQuantized(OwmMathLatexFtScienceCombined): + MODEL_PATH = FastTextScienceTaggerQuantized.MODEL_PATH # pyright: ignore + + +@TaggerRegistry.add("owmV2_FTsciV1_comb_lth_qt") +class OwmMathLatexFtScienceCombinedLowThresholdQuantized(OwmMathLatexFtScienceCombinedLowThreshold): + MODEL_PATH = FastTextScienceTaggerQuantized.MODEL_PATH # pyright: ignore + + +@TaggerRegistry.add("owmV2_FTsciV1_comb_hth_qt") +class OwmMathLatexFtScienceCombinedHighThresholdQuantized(OwmMathLatexFtScienceCombinedHighThreshold): + MODEL_PATH = FastTextScienceTaggerQuantized.MODEL_PATH # pyright: ignore + +@TaggerRegistry.add("owmV2_FTsciV1_comb_vhth_qt") +class OwmMathLatexFtScienceCombinedVeryHighThresholdQuantized(OwmMathLatexFtScienceCombinedVeryHighThreshold): + MODEL_PATH = FastTextScienceTaggerQuantized.MODEL_PATH # pyright: ignore + +@TaggerRegistry.add("owmV2_FTsciV1_comb_vvhth_qt") # FST2 +class OwmMathLatexFtScienceCombinedVeryVeryHighThresholdQuantized(OwmMathLatexFtScienceCombinedVeryVeryHighThreshold): + MODEL_PATH = "~/test_analyze/model_09_06_quantized.bin" # pyright: ignore \ No newline at end of file diff --git a/python/dolma/tokenizer/__main__.py b/python/dolma/tokenizer/__main__.py new file mode 100644 index 00000000..c2ef501c --- /dev/null +++ b/python/dolma/tokenizer/__main__.py @@ -0,0 +1,32 @@ +import argparse + +from .executor import tokenize_in_parallel + + +def parse_args() -> argparse.Namespace: + ap = argparse.ArgumentParser() + ap.add_argument("--sources", nargs="+", required=True) + ap.add_argument("--destination", required=True) + ap.add_argument("--tokenizer-id", type=str, default="allenai/eleuther-ai-gpt-neox-20b-pii-special") + ap.add_argument("--metadata-dir", type=str, default=None) + ap.add_argument("--num-readers", type=int, default=1) + ap.add_argument("--num-writers", type=int, default=1) + ap.add_argument("--max-size", type=int, default=1024 * 1024 * 1024) + return ap.parse_args() + + +def main(): + args = parse_args() + tokenize_in_parallel( + sources=args.sources, + destination=args.destination, + tokenizer_name_or_path=args.tokenizer_id, + metadata_dir=args.metadata_dir, + num_readers=args.num_readers, + num_writers=args.num_writers, + max_size=args.max_size, + ) + + +if __name__ == "__main__": + main() diff --git a/python/dolma/tokenizer/executor.py b/python/dolma/tokenizer/executor.py index 7949c580..59252ea6 100644 --- a/python/dolma/tokenizer/executor.py +++ b/python/dolma/tokenizer/executor.py @@ -274,8 +274,7 @@ def __call__(self, num_readers: Optional[int] = None, **process_single_kwargs: A ) # finally run the processors - fn = self._debug_run_all if self.debug else self._multiprocessing_run_all - fn( + self._run_all( all_source_paths=source_indices, all_destination_paths=all_destination_paths, all_metadata_paths=all_metadata_path, diff --git a/python/dolma/warc/iterator.py b/python/dolma/warc/iterator.py new file mode 100644 index 00000000..2e496d39 --- /dev/null +++ b/python/dolma/warc/iterator.py @@ -0,0 +1,153 @@ +import random +import time +from functools import reduce +from io import BytesIO, TextIOWrapper +from typing import TYPE_CHECKING, Generator, List, Optional, Union + +import smart_open +from necessary import necessary + +from ..core.loggers import get_logger + +with necessary("fastwarc", soft=True) as FASTWARC_AVAILABLE: + if FASTWARC_AVAILABLE or TYPE_CHECKING: + from fastwarc.stream_io import ( # pylint: disable=no-name-in-module + GZipStream, + LZ4Stream, + ) + from fastwarc.warc import ( # pylint: disable=no-name-in-module + ArchiveIterator, + WarcRecord, + WarcRecordType, + ) + + +class BackoffWarcIterator: + def __init__( + self, + path: str, + max_time: Optional[float] = None, + min_wait: float = 1.0, + max_tries: int = 10, + max_wait: Optional[float] = None, + record_types: Optional[List[Union[str, "WarcRecordType"]]] = None, + ): + self.path = path + self.max_time = max_time + self.max_tries = max_tries + self.max_wait = max_wait or float("inf") + self.min_wait = min_wait + self.logger = get_logger(self.__class__.__name__) + + self._file_object: Optional[Union[TextIOWrapper, BytesIO]] = None + self._start_time = float("-inf") + self._attempt = 0 + self._location = 0 + + record_types = record_types or ["response", "warcinfo"] + self.record_types = [WarcRecordType[r] if isinstance(r, str) else r for r in record_types] + + def _open(self): + self._attempt += 1 + + # close any previous file object + self.close() + + if self.path.endswith(".lz4"): + warc_stream = smart_open.open(self.path, "rb", compression="disable") + warc_stream.seek(self._location) + self._file_object = LZ4Stream(warc_stream) + elif self.path.endswith(".gz"): + warc_stream = smart_open.open(self.path, "rb", compression="disable") + warc_stream.seek(self._location) + self._file_object = GZipStream(warc_stream) + else: + obj = smart_open.open(self.path, "rt") + obj.seek(self._location) + self._file_object = obj + + def close(self): + if self._file_object is not None: + self._file_object.close() + + def wait(self): + # exponential backoff with jitter + # https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ + wait_time = random.uniform(0, min(2**self._attempt * self.min_wait, self.max_wait)) + + # inform user, then sleep + self.logger.warning("Failed to read %s, retrying in %.2f seconds.", self.path, wait_time) + time.sleep(wait_time) + + def should_raise(self): + if self._attempt >= self.max_tries: + self.logger.error("Failed to read %s after %d tries.", self.path, self._attempt) + return True + + if self.max_time and (self._start_time + time.time()) > self.max_time: + self.logger.error("Failed to read %s after %.2f seconds.", self.path, self.max_time) + return True + + def __enter__(self): + self._open() + self._start_time = time.time() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + self._file_object = None + self._location = 0 + self._start_time = float("-inf") + self._attempt = 0 + + def __iter__(self) -> Generator["WarcRecord", None, None]: + if self._file_object is None: + raise OSError("File object must be opened before iterating.") + + while True: + try: + it = ArchiveIterator(self._file_object, record_types=reduce(lambda a, b: a | b, self.record_types)) + for record in it: + self._location = self._file_object.tell() + yield record + return + except Exception as exp: + if self.should_raise(): + self.logger.exception("Failed to read %s.", self.path) + raise exp + + self.wait() + self._open() + + +class SimpleWarcIterator: + def __init__(self, path: str, record_types: Optional[List[Union[str, "WarcRecordType"]]] = None): + self.path = path + self.record_types = [ + WarcRecordType[r] if isinstance(r, str) else r for r in (record_types or ["response", "warcinfo"]) + ] + self._fobj: Optional[Union[BytesIO, TextIOWrapper]] = None + self._it: Optional["ArchiveIterator"] = None + + def __enter__(self): + if self.path.endswith(".lz4"): + warc_stream = smart_open.open(self.path, "rb", compression="disable") + self._fobj = LZ4Stream(warc_stream) + elif self.path.endswith(".gz"): + warc_stream = smart_open.open(self.path, "rb", compression="disable") + self._fobj = GZipStream(warc_stream) + else: + self._fobj = smart_open.open(self.path, "rt") + self._it = ArchiveIterator(self._fobj, record_types=reduce(lambda a, b: a | b, self.record_types)) + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self._fobj is not None: + self._fobj.close() + self._fobj = None + self._it = None + + def __iter__(self) -> Generator["WarcRecord", None, None]: + if self._it is None: + raise OSError("File object must be opened before iterating.") + yield from self._it diff --git a/python/dolma/warc/linearizers.py b/python/dolma/warc/linearizers.py index a99c0775..5f592b59 100644 --- a/python/dolma/warc/linearizers.py +++ b/python/dolma/warc/linearizers.py @@ -1,6 +1,7 @@ import logging +import re from abc import abstractmethod -from typing import TYPE_CHECKING, Iterable, Optional, Type, Union +from typing import TYPE_CHECKING, Iterable, Optional, Type from necessary import necessary @@ -9,21 +10,37 @@ with necessary("trafilatura", soft=True) as TRAFILATURA_AVAILABLE: if TRAFILATURA_AVAILABLE or TYPE_CHECKING: - import trafilatura # noqa: F401 - import trafilatura.meta # noqa: F401 + import trafilatura # pylint: disable=import-error # pyright:ignore + import trafilatura.meta # pylint: disable=import-error # pyright:ignore + import trafilatura.utils # pylint: disable=import-error # pyright:ignore with necessary("resiliparse", soft=True) as RESILIPARSE_AVAILABLE: if RESILIPARSE_AVAILABLE or TYPE_CHECKING: - from resiliparse.extract.html2text import extract_plain_text # noqa: F401 - from resiliparse.parse.encoding import detect_encoding # noqa: F401 - from resiliparse.parse.html import HTMLTree # noqa: F401 + from resiliparse.extract.html2text import ( # pylint: disable=no-name-in-module # pyright:ignore + extract_plain_text, + ) + from resiliparse.parse.encoding import ( # pylint: disable=no-name-in-module # pyright:ignore + detect_encoding, + ) + from resiliparse.parse.html import ( # pylint: disable=no-name-in-module # pyright:ignore + HTMLTree, + ) + +with necessary("text_extract", soft=True) as OPENWEBMATH_AVAILABLE: + if OPENWEBMATH_AVAILABLE or TYPE_CHECKING: + from text_extract.extract import ( + extract_text as owm_extract_text, # pylint: disable=import-error # pyright:ignore + ) + from text_extract.utils import ( + Config as OpenWebMathConfig, # pylint: disable=import-error # pyright:ignore + ) class BaseLinearizer: """A base class for linearizers, i.e. tools to turn HTML into text.""" @abstractmethod - def linearize(self, content: Union[str, bytes]) -> str: + def linearize(self, content: bytes, encoding: Optional[str] = None) -> str: pass @@ -42,7 +59,7 @@ def __init__( links: bool = False, form_fields: bool = False, noscript: bool = False, - comments: bool = True, + comments: bool = False, skip_elements: Optional[Iterable[str]] = None, ) -> None: assert RESILIPARSE_AVAILABLE, raise_warc_dependency_error("resiliparse") @@ -57,8 +74,8 @@ def __init__( self.comments = comments self.skip_elements = skip_elements - def linearize(self, content: Union[str, bytes]) -> str: - # html (HTMLTree or str) – HTML as DOM tree or Unicode string + def linearize(self, content: bytes, encoding: Optional[str] = None) -> str: + # html (HTMLTree or str) – HTML as DOM tree or Unicode string # preserve_formatting (bool) – preserve basic block-level formatting # main_content (bool) – apply simple heuristics for extracting only “main-content” elements # list_bullets (bool) – insert bullets / numbers for list items @@ -68,11 +85,7 @@ def linearize(self, content: Union[str, bytes]) -> str: # noscript (bool) – extract contents of